diff --git a/conf/docker/.env.setup b/conf/docker/.env.setup index 434c5b3..e4f75e5 100644 --- a/conf/docker/.env.setup +++ b/conf/docker/.env.setup @@ -310,12 +310,16 @@ MC_OBSERVABILITY_MCP_MARIA_PORT=8001 MC_OBSERVABILITY_MCP_INFLUX_PORT=8002 # Host-side names for values the observability images read under their own -# fixed env keys (GRAFANA_API_KEY / OPENAI_API_KEY / OLLAMA_BASE_URL). +# fixed env keys (GRAFANA_API_KEY / OPENAI_API_KEY / OLLAMA_BASE_URL / +# LLM_CREDENTIAL_KEY). # docker-compose.yaml maps these prefixed variables onto those keys so the # generic names never collide with the host environment. MC_OBSERVABILITY_GRAFANA_API_KEY= MC_OBSERVABILITY_OPENAI_API_KEY= MC_OBSERVABILITY_OLLAMA_BASE_URL=http://210.207.104.224:11434 +# Fernet key that encrypts stored LLM API keys. Leave empty to let insight +# generate one into the insight_secrets volume on first start. +MC_OBSERVABILITY_LLM_CREDENTIAL_KEY= # MC-WEB-CONSOLE MC_WEB_CONSOLE_POSTGRES_DB=mcwebconsoledbdev diff --git a/conf/docker/conf/mc-observability/mariadb/maria_init.sql b/conf/docker/conf/mc-observability/mariadb/maria_init.sql index e71b89d..7c38fc5 100644 --- a/conf/docker/conf/mc-observability/mariadb/maria_init.sql +++ b/conf/docker/conf/mc-observability/mariadb/maria_init.sql @@ -21,36 +21,67 @@ CREATE TABLE `mc_o11y_insight_anomaly_setting` ( PRIMARY KEY (`SEQ`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; -CREATE TABLE `mc_o11y_insight_llm_api_key` ( +CREATE TABLE `mc_o11y_insight_llm_connection` ( `SEQ` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `NAME` varchar(100) NOT NULL, `PROVIDER` varchar(20) NOT NULL, - `API_KEY` text DEFAULT NULL, `BASE_URL` text DEFAULT NULL, - PRIMARY KEY (`SEQ`) + `API_KEY_ENCRYPTED` text DEFAULT NULL, + `DEFAULT_MODEL` varchar(255) DEFAULT NULL, + `IS_DEFAULT` boolean NOT NULL DEFAULT 0, + `ENABLED` boolean NOT NULL DEFAULT 1, + `REGDATE` timestamp NOT NULL DEFAULT current_timestamp(), + PRIMARY KEY (`SEQ`), + UNIQUE KEY `uk_llm_connection_name` (`NAME`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; CREATE TABLE `mc_o11y_insight_chat_session` ( `SEQ` bigint(20) NOT NULL AUTO_INCREMENT, `USER_ID` varchar(100) NOT NULL DEFAULT '1', `SESSION_ID` varchar(100) NOT NULL, + `CONNECTION_ID` bigint(20) unsigned DEFAULT NULL, + `ANALYSIS_TYPE` varchar(20) NOT NULL DEFAULT 'log', `PROVIDER` varchar(20) NOT NULL, - `MODEL_NAME` varchar(20) NOT NULL, + `MODEL_NAME` varchar(255) NOT NULL, `REGDATE` timestamp NOT NULL DEFAULT current_timestamp(), - PRIMARY KEY (`SEQ`) + PRIMARY KEY (`SEQ`), + KEY `idx_chat_session_connection_id` (`CONNECTION_ID`), + CONSTRAINT `fk_chat_session_connection` + FOREIGN KEY (`CONNECTION_ID`) + REFERENCES `mc_o11y_insight_llm_connection` (`SEQ`) + ON DELETE RESTRICT ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; CREATE TABLE `mc_o11y_insight_server_error_analysis` ( `ID` bigint(20) unsigned NOT NULL AUTO_INCREMENT, - `TRACE_ID` varchar(64) DEFAULT NULL COMMENT '대표 trace_id, NULL이면 unique dedup 미적용', + `TRACE_ID` varchar(64) DEFAULT NULL COMMENT '분석 범위의 대표 trace_id', `SESSION_ID` varchar(100) NOT NULL COMMENT '연결된 채팅 세션 ID', - `STATUS` varchar(20) NOT NULL DEFAULT 'PENDING' COMMENT 'PENDING, RUNNING, SUCCEEDED, FAILED, PARTIAL', + `STATUS` varchar(20) NOT NULL DEFAULT 'RUNNING' COMMENT 'RUNNING, SUCCEEDED, FAILED, PARTIAL', `SUMMARY` text DEFAULT NULL COMMENT '목록/상세 화면에 표시할 최종 요약', + `REQUEST_JSON` json DEFAULT NULL COMMENT '검증된 RCA 요청 원문', `DETAIL_JSON` json DEFAULT NULL COMMENT '위험도, 근거 요약, trace/log 요약, 추천 조치, 오류 정보', `CREATED_AT` timestamp NOT NULL DEFAULT current_timestamp(), `UPDATED_AT` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), PRIMARY KEY (`ID`), - UNIQUE KEY `uk_server_error_trace_id` (`TRACE_ID`), + KEY `ix_server_error_trace_id` (`TRACE_ID`), KEY `idx_server_error_session_id` (`SESSION_ID`), KEY `idx_server_error_status` (`STATUS`), KEY `idx_server_error_updated_at` (`UPDATED_AT`) ) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; + +CREATE TABLE `mc_o11y_insight_rca_schedule` ( + `ID` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `NAME` varchar(100) NOT NULL COMMENT '사용자 식별용 이름', + `ENABLED` boolean NOT NULL DEFAULT 1 COMMENT '향후 실행 활성화 여부', + `INTERVAL_MINUTES` int(10) unsigned NOT NULL COMMENT '완료 후 다음 실행까지의 분 단위 간격', + `REQUEST_JSON` json NOT NULL COMMENT '실행할 RCA 요청 본문', + `STATUS` varchar(20) NOT NULL DEFAULT 'IDLE' COMMENT 'IDLE, RUNNING, SUCCEEDED, PARTIAL, FAILED', + `LAST_EXECUTION` timestamp NULL DEFAULT NULL COMMENT '마지막 실행 종료 UTC 시각', + `NEXT_EXECUTION` timestamp NULL DEFAULT NULL COMMENT '다음 실행 예정 UTC 시각', + `LAST_ANALYSIS_ID` bigint(20) unsigned DEFAULT NULL COMMENT '마지막 RCA 분석 레코드 ID', + `LAST_ERROR` text DEFAULT NULL COMMENT '마지막 실행 실패 원인', + `CREATED_AT` timestamp NOT NULL DEFAULT current_timestamp(), + `UPDATED_AT` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp(), + PRIMARY KEY (`ID`), + KEY `ix_rca_schedule_due` (`ENABLED`, `NEXT_EXECUTION`, `STATUS`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; diff --git a/conf/docker/docker-compose.yaml b/conf/docker/docker-compose.yaml index 455705e..b6a09b9 100644 --- a/conf/docker/docker-compose.yaml +++ b/conf/docker/docker-compose.yaml @@ -18,6 +18,7 @@ networks: volumes: grafana_shared_config: shared_logs: + insight_secrets: services: ##### MC-INFRA-CONNECTOR ######################################################################################################################### @@ -1246,7 +1247,7 @@ services: <<: *default-health-check ##### MC-OBSERVABILITY ######################################################################################################################### mc-observability-manager: - image: cloudbaristaorg/mc-observability:0.6.1 + image: cloudbaristaorg/mc-observability:0.6.2 container_name: mc-observability-manager restart: unless-stopped networks: @@ -1286,7 +1287,7 @@ services: start_period: 5m mc-observability-front: - image: cloudbaristaorg/mc-observability-front:0.6.1 + image: cloudbaristaorg/mc-observability-front:0.6.2 container_name: mc-observability-front restart: unless-stopped networks: @@ -1298,7 +1299,7 @@ services: condition: service_started mc-observability-infra: - image: cloudbaristaorg/mc-observability-infra:0.6.1 + image: cloudbaristaorg/mc-observability-infra:0.6.2 container_name: mc-observability-infra restart: unless-stopped networks: @@ -1423,7 +1424,7 @@ services: init: true mc-observability-influx: - image: cloudbaristaorg/mc-observability-influx:0.6.1 + image: cloudbaristaorg/mc-observability-influx:0.6.2 container_name: mc-observability-influx restart: unless-stopped networks: @@ -1465,7 +1466,7 @@ services: - mc-observability-network mc-observability-influx-2: - image: cloudbaristaorg/mc-observability-influx:0.6.1 + image: cloudbaristaorg/mc-observability-influx:0.6.2 container_name: mc-observability-influx-2 restart: unless-stopped networks: @@ -1641,7 +1642,7 @@ services: init: true mc-observability-grafana: - image: cloudbaristaorg/mc-observability-grafana:0.6.1 + image: cloudbaristaorg/mc-observability-grafana:0.6.2 container_name: mc-observability-grafana restart: unless-stopped networks: @@ -1676,7 +1677,7 @@ services: fi mc-observability-insight: - image: cloudbaristaorg/mc-observability-insight:0.6.1 + image: cloudbaristaorg/mc-observability-insight:0.6.2 container_name: mc-observability-insight restart: always networks: @@ -1688,8 +1689,10 @@ services: environment: - TZ=Asia/Seoul - OLLAMA_BASE_URL=${MC_OBSERVABILITY_OLLAMA_BASE_URL:-http://210.207.104.224:11434} + - LLM_CREDENTIAL_KEY=${MC_OBSERVABILITY_LLM_CREDENTIAL_KEY:-} volumes: - shared_logs:/mc-insight/log:rw + - insight_secrets:/mc-insight/secrets:rw depends_on: mc-observability-maria: condition: service_healthy @@ -1703,7 +1706,7 @@ services: start_period: 10s mc-observability-insight-scheduler: - image: cloudbaristaorg/mc-observability-insight-scheduler:0.6.1 + image: cloudbaristaorg/mc-observability-insight-scheduler:0.6.2 container_name: mc-observability-insight-scheduler restart: always ports: @@ -1713,6 +1716,8 @@ services: depends_on: mc-observability-maria: condition: service_healthy + mc-observability-insight: + condition: service_healthy environment: - TZ=Asia/Seoul - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=mysql+mysqldb://mc-agent:mc-agent@mc-observability-maria:3306/mc_airflow @@ -1739,10 +1744,17 @@ services: airflow connections add --conn-type http --conn-host mc-observability-insight --conn-schema http --conn-port 9001 api_base_url airflow connections add --conn-type mysql --conn-host mc-observability-maria --conn-schema mc_observability --conn-login mc-agent --conn-password mc-agent --conn-port 3306 mcmp_db + # Server error watch: the RCA worker searches Tempo before running an analysis + airflow connections add --conn-type http --conn-host mc-observability-tempo --conn-schema http --conn-port 3200 tempo_url + # Bound how many scheduled RCA analyses run at once; each one is long and LLM-backed + airflow pools set rca_analysis 4 'Concurrent scheduled RCA analyses' + # Reload & Run dags airflow dags reserialize airflow dags unpause anomaly_detection airflow dags unpause down_sampling + airflow dags unpause rca_schedule_dispatcher + airflow dags unpause rca_schedule_worker # Start airflow airflow scheduler & airflow webserver -p 9002 @@ -1782,7 +1794,7 @@ services: - mc-observability-network mc-observability-mcp-maria: - image: cloudbaristaorg/mc-observability-mcp-mariadb:0.6.1 + image: cloudbaristaorg/mc-observability-mcp-mariadb:0.6.2 container_name: mc-observability-mcp-maria restart: always networks: @@ -1806,7 +1818,7 @@ services: - OPENAI_API_KEY=${MC_OBSERVABILITY_OPENAI_API_KEY:-} mc-observability-mcp-influx: - image: cloudbaristaorg/mc-observability-mcp-influxdb:0.6.1 + image: cloudbaristaorg/mc-observability-mcp-influxdb:0.6.2 container_name: mc-observability-mcp-influx restart: always ports: @@ -1819,6 +1831,8 @@ services: environment: - TZ=Asia/Seoul - INFLUXDB_URL=http://mc-observability-influx:8086 + # Infra and node metrics can reside on either server; the MCP queries both. + - INFLUXDB_URLS=http://mc-observability-influx:8086,http://mc-observability-influx-2:8086 - INFLUXDB_USER=mc-agent - INFLUXDB_PASSWORD=mc-agent - INFLUXDB_DATABASE=mc-observability