AICE 대비 2탄

탐색적 데이터 분석

필요한 라이브러리 임포트

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

데이터 불러오기

df = pd.read_csv('파일명.csv')
df = pd.read_json('파일명.json')

데이터 확인

df.head()
df.tail()
df.shape

df.columns
df.dtypes
df.info()
df.describe(include, exclude)
df.corr(numeric_only=False)

df.sort_values(by, axis=0, ascending=True)

df.nunique()
df['컬럼명'].unique()
df['컬럼명'].value_counts(normalize=False)

df['컬럼명'].isin(values)
df['컬럼명'].between(left, right, inclusive='both')

df.drop(columns=['컬럼명'])
df.replace(to_replace, value)

pd.concat(objs, axis=0, join='outer')
pd.merge(left, right, how='inner', on)

데이터 시각화

# 단변량
sns.histplot(x='컬럼명', data=df)
sns.kdeplot(x='컬럼명', data=df)
sns.boxplot(x='컬럼명', data=df)
sns.countplot(x='컬럼명', data=df)

# 이변량
sns.scatterplot(x='컬럼명', y='컬럼명', data=df)
sns.pairplot(x='컬럼명', y='컬럼명', data=df)
sns.jointplot(x='컬럼명', y='컬럼명', data=df)

# 상관관계 히트맵
sns.heatmap(df.corr(), annot=True, cmap='Blues')

plt.subplot(nrows, ncols, index)
plt.axhline(y=0)  # 수평선
plt.axvline(x=0)  # 수직선

plt.title('label')
plt.xlabel('xlabel')
plt.ylabel('ylabel')

plt.grid()    # 격자
plt.legend()  # 범례
plt.tight_layout
plt.show()

 

데이터 전처리

결측치 처리

# 결측치 확인
df.isna().sum()

# 평균으로 채우기
df['컬럼명'] = df['컬럼명'].fillna(df['컬럼명'].mean())

# 최빈값으로 채우기
df['컬럼명'] = df['컬럼명'].fillna(df['컬럼명'].mode()[0])

# 삭제하기
df = df.dropna()        # 결측치 포함된 행 제거
df = df.dropna(axis=1)  # 결측치 포함된 열 제거
df = df.dropna(subset, axis=0, how='any')

중복 제거

# 중복 행 확인
df.duplicated()

# 중복 제거
df = df.drop_duplicates()

라벨 인코딩 / 원핫 인코딩

# object 타입 컬럼 선택
cols = df.select_dtypes(['object']).columns

# 라벨 인코딩
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
df['컬럼명'] = le.fit_transform(df['컬럼명'])

# 원핫 인코딩
df = pd.get_dummies(df, columns=['컬럼명'], drop_first=True)

X, y 데이터 분리

X = df.drop(columns=['target'])
y = df['target']

데이터 분할

from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.2,       # 검증 비율
    random_state=42,     # 재현성 있는 분할
    stratify=y           # 분류 문제에서 비율 유지하고 싶을 때 사용
)

정규화 / 표준화

from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler

# MinMaxScaler
scaler = MinMaxScaler()
df_scaled = scaler.fit_transform(df)

# StandardScaler
scaler = StandardScaler()
df_scaled = scaler.fit_transform(df)

# RobustScaler
scaler = RobustScaler()
df_scaled = scaler.fit_transform(df)

 

머신러닝 모델링

# 불러오기
from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from xgboost import XGBRegressor, XGBClassifier
from lightgbm import LGBMRegressor, LGBMClassifier

# 선언하기
model_1 = DecisionTreeRegressor()
model_2 = RandomForestClassifier()

# 학습하기
model_1.fit(X_train, y_train)
model_2.fit(X_train, y_train)

머신러닝 모델 성능평가

from sklearn.metrics import *

# 예측하기
y_pred_1 = model_1.predict(X_test)
y_pred_2 = model_2.predict(X_test)

# 회귀 모델 평가하기
mse = mean_squared_error(y_test, y_pred_1)
rmse = root_mean_squared_error(y_test, y_pred_1)
mae = mean_absolute_error(y_test, y_pred_1)
mape = mean_absolute_percentage_error(y_test, y_pred_1)
r2 = r2_score(y_test, y_pred_1)

# 분류 모델 평가하기
confusion_matrix = confusion_matrix(y_test, y_pred_2)
accuracy = accuracy_score(y_test, y_pred_2)
precision = precision_score(y_test, y_pred_2)
recall = recall_score(y_test, y_pred_2)
f1 = f1_score(y_test, y_pred_2)

 

딥러닝 모델링

import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Input, Dense, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

model = Sequential([
    Input(shape=(X_train.shape[1],)),
    Dense(128, activation='relu'),
    Dropout(0.2),
    Dense(64, activation='relu'),
    Dropout(0.2),
    Dense(32, activation='relu'),
    Dense(1, activation=None)  # 회귀: None, 이진분류: 'sigmoid', 다중분류: 'softmax'
])

model.compile(
    optimizer='adam',
    loss='mse',      # 회귀: 'mse', 이진분류: 'binary_crossentropy', 다중분류: 'categorical_crossentropy'
    metrics=['mae']  # 'mae', 'mse', 'accuracy', ...
)

es = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)
mc = ModelCheckpoint('best_model.keras', monitor='val_loss', save_best_only=True)

history = model.fit(
    X_train, y_train,
    validation_data=(X_valid, y_valid),
    epochs=30,
    batch_size=16,
    callbacks=[es, mc],
    verbose=0
)

딥러닝 모델 성능평가

from sklearn.metrics import *

# 학습곡선
plt.plot(history.history['mse'], label='mse')  # color='royalblue', linewidth=2
plt.plot(history.history['val_mse'], label='val_mse')  # color='tomato', linewidth=2
plt.title('Model MSE')
plt.xlabel('Epochs')
plt.ylabel('MSE')
plt.grid()    # 격자
plt.legend()  # 범례
plt.tight_layout()
plt.show()

# Confusion Matrix
y_pred = model.predict(X_text)
cm = confusion_matrix(y_test, y_pred)
disp = ConfusionMatrixDisplay(confusion_matrix=cm)
disp.plot()  # cmap='Blues', values_format='d', colorbar=False
plt.tight_layout()
plt.show()

# Classification Report
y_pred = model.predict(X_text)
print(classification_report(y_test, y_pred)