# Preview Data
data.shape

data.describe

data.head(10)

data.tail(10)

data['Description'].value_counts()

# Count the Impact Distribution
cnt_impact = data['Impact'].value_counts()
print(cnt_impact)

# Create a bar plot
cnt_impact.plot(kind='bar', color=['orange', 'green', 'red', 'grey'])
plt.title('Impact Distribution')
plt.xlabel('Impact')
plt.ylabel('Count')
plt.xticks(rotation=0)
plt.show()

# Show the HIGH Impact ONLY
High_Impact = data[data['Impact']=='H'][['Date', 'Time', 'Description', 'Actual', 'Forecast']]
print(High_Impact)

# Show the MEDIUM Impact ONLY
Medium_Impact = data[data['Impact']=='M'][['Date', 'Time', 'Description', 'Actual', 'Forecast', 'Previous']]
print(Medium_Impact)

# Check for NaN in the entire DataFrame
nan_check = data_clean.isnull().values.any()
# Count NaN values in each column
nan_count_per_column = data_clean.isnull().sum()
print(nan_count_per_column)

data_clean.tail(5)

data_clean['Sentiment'].tail(10)

# Count the results to know how many for Neutral, Postive dan Negative sentiments.
cnt_sentiment = data_clean['Sentiment'].value_counts()
print(cnt_sentiment)

# Create a bar plot
import matplotlib.pyplot as plt

cnt_sentiment.plot(kind='bar', color=['black', 'red', 'green'])
plt.title('SENTIMENT DISTRIBUTION')
plt.xlabel('SENTIMENT')
plt.ylabel('COUNT')
plt.xticks(rotation=0)
plt.show()

# Show the Positive sentiments ONLY
positive_sentiment = data_clean[data_clean['Sentiment']=='positive'][['Description', 'Actual', 'Forecast', 'Sentiment']]
print(positive_sentiment)

# Show the negative sentiments ONLY
negative_sentiment = data_clean[data_clean['Sentiment']=='negative'][['Description', 'Actual', 'Forecast', 'Sentiment']]
print(negative_sentiment)

data_clean.describe().T

data_clean['Description'].value_counts()

# Keep and Re-check for NaNs values.
data_clean.dropna(subset=['Actual', 'Forecast', 'Previous'], inplace=True)
# Check for NaN in the entire DataFrame
nan_check = data_clean.isnull().values.any()
# Count NaN values in each column
nan_count_per_column = data_clean.isnull().sum()
print(nan_count_per_column)

#Data splitting
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0)

print('X_train : ', X_train.shape)
print('X_test : ', X_test.shape)
print('y_train : ', y_train.shape)
print('y_test : ', y_test.shape)

# Define a model and fitting the model by using LOGISTIC REGRESSION model
from sklearn.linear_model import LogisticRegression

model_LR = LogisticRegression(max_iter=2000)
model_LR.fit(X_train, y_train) # --- Training Process

# Making predictions on the test set
y_pred_LR = model_LR.predict(X_test) # -- Testing Process

# Model evaluation
accuracy_LR = accuracy_score(y_test, y_pred_LR)
print(f"Accuracy LR: {accuracy_LR:.2f}")

# Classification report showing precision, recall, F1-score, etc.
print("Classification Report LR:")
print(classification_report(y_test, y_pred_LR))

# Visualize the classification report

# Create a classification report
report_LR = classification_report(y_test, y_pred_LR, output_dict=True)

# Extract precision, recall, and F1-score from the report
classes_LR = list(report_LR.keys())[:-3]  # Extract class names, excluding 'accuracy', 'macro avg', 'weighted avg'

precision_LR = [report_LR[c]['precision'] for c in classes_LR]
recall_LR = [report_LR[c]['recall'] for c in classes_LR]
f1_score_LR = [report_LR[c]['f1-score'] for c in classes_LR]

# Create a bar plot for precision, recall, and F1-score
x = np.arange(len(classes_LR))  # X-axis locations for the bars
width = 0.3  # Width of the bars

fig, ax = plt.subplots(figsize=(10, 6))

# Plot precision, recall, and F1-score for each class
ax.bar(x - width, precision_LR, width, label='Precision LR')
ax.bar(x, recall_LR, width, label='Recall LR')
ax.bar(x + width, f1_score_LR, width, label='F1-score LR')

ax.set_xticks(x)
ax.set_xticklabels(classes_LR)
ax.set_xlabel('Classes')
ax.set_ylabel('Score')
ax.set_title('Classification Report Metrics - Logistic Regression Model')
ax.legend()

plt.tight_layout()
plt.show()

# Creating and fitting the RANDOM FOREST
from sklearn.ensemble import RandomForestClassifier

model_RF = RandomForestClassifier(n_estimators=100, random_state=42)
model_RF.fit(X_train, y_train)  # Training the model

# Making predictions on the test set
y_pred_RF = model_RF.predict(X_test)

# Model evaluation
accuracy_RF = accuracy_score(y_test, y_pred_RF)
print(f"Accuracy RF: {accuracy_RF:.2f}")

# Classification report showing precision, recall, F1-score, etc.
print("Classification Report:")
print(classification_report(y_test, y_pred_RF))

# Create a classification report
report_RF = classification_report(y_test, y_pred_RF, output_dict=True)

# Extract precision, recall, and F1-score from the report
classes_RF = list(report_RF.keys())[:-3]  # Extract class names, excluding 'accuracy', 'macro avg', 'weighted avg'

precision_RF = [report_RF[c]['precision'] for c in classes_RF]
recall_RF = [report_RF[c]['recall'] for c in classes_RF]
f1_score_RF = [report_RF[c]['f1-score'] for c in classes_RF]

# Create a bar plot for precision, recall, and F1-score
x = np.arange(len(classes_RF))  # X-axis locations for the bars
width = 0.3  # Width of the bars

fig, ax = plt.subplots(figsize=(10, 6))

# Plot precision, recall, and F1-score for each class
ax.bar(x - width, precision_RF, width, label='Precision RF')
ax.bar(x, recall_RF, width, label='Recall RF')
ax.bar(x + width, f1_score_RF, width, label='F1-score RF')

ax.set_xticks(x)
ax.set_xticklabels(classes_RF)
ax.set_xlabel('Classes')
ax.set_ylabel('Score')
ax.set_title('Classification Report Metrics - Random Forest Model')
ax.legend()

plt.tight_layout()
plt.show()

# predict with RF model
pred_sentiment_RF = model_RF.predict(df_input)
print(pred_sentiment_RF)

# predict with LR model
pred_sentiment_LR = model_LR.predict(df_input)
print(pred_sentiment_LR)

from sklearn.compose import ColumnTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split

# Splitting the data into features and target
X = data_clean.drop('Sentiment', axis=1)  # Features
y = data_clean['Sentiment']  # Target

# Splitting the data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Preprocessing for numerical features
numeric_features = ['Actual', 'Forecast', 'Previous']
numeric_transformer = StandardScaler()

# Preprocessing for text features
text_features = 'Description'
text_transformer = CountVectorizer()

# Combine preprocessing for numerical and text features
preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, numeric_features),
        ('text', text_transformer, text_features)
    ])

# Create the pipeline with preprocessing and model
pipeline = Pipeline(steps=[('preprocessor', preprocessor),
                           ('classifier', RandomForestClassifier())])

# Fit the model
pipeline.fit(X_train, y_train)

# Predictions
predictions = pipeline.predict(X_test)

# Evaluate the model
from sklearn.metrics import accuracy_score, classification_report

accuracy_pipeline = accuracy_score(y_test, predictions)
report_pipeline = classification_report(y_test, predictions)

print(f"Accuracy: {accuracy_pipeline}")
print(f"Classification Report:\n{report_pipeline}")

# Create a classification report
report_pipeline = classification_report(y_test, predictions, output_dict=True)

# Extract precision, recall, and F1-score from the report
classes_pipeline = list(report_pipeline.keys())[:-3]  # Extract class names, excluding 'accuracy', 'macro avg', 'weighted avg'

precision_pipeline = [report_pipeline[c]['precision'] for c in classes_pipeline]
recall_pipeline = [report_pipeline[c]['recall'] for c in classes_pipeline]
f1_score_pipeline = [report_pipeline[c]['f1-score'] for c in classes_pipeline]

# Create a bar plot for precision, recall, and F1-score
x = np.arange(len(classes_pipeline))  # X-axis locations for the bars
width = 0.3  # Width of the bars

fig, ax = plt.subplots(figsize=(10, 6))

# Plot precision, recall, and F1-score for each class
ax.bar(x - width, precision_pipeline, width, label='Precision pipeline')
ax.bar(x, recall_pipeline, width, label='Recall pipeline')
ax.bar(x + width, f1_score_pipeline, width, label='F1-score pipeline')

ax.set_xticks(x)
ax.set_xticklabels(classes_pipeline)
ax.set_xlabel('Classes')
ax.set_ylabel('Score')
ax.set_title('Classification Report Metrics - Pipeline Model')
ax.legend()

plt.tight_layout()
plt.show()

# predict with pipeline model
pred_sentiment_pipeline = pipeline.predict(df_input)
print(pred_sentiment_pipeline)

data_clean

data_clean.plot(kind='scatter', x='Forecast', y='Previous', s=32, alpha=.8)
plt.gca().spines[['top', 'right',]].set_visible(False)

def _plot_series(series, series_name, series_index=0):
  from matplotlib import pyplot as plt
  import seaborn as sns
  palette = list(sns.palettes.mpl_palette('Dark2'))
  xs = series['Date']
  ys = series['Actual']

  plt.plot(xs, ys, label=series_name, color=palette[series_index % len(palette)])

fig, ax = plt.subplots(figsize=(10, 5.2), layout='constrained')
df_sorted = data_clean.sort_values('Date', ascending=True)
for i, (series_name, series) in enumerate(df_sorted.groupby('Sentiment')):
  _plot_series(series, series_name, i)
  fig.legend(title='Sentiment', bbox_to_anchor=(1, 1), loc='upper left')
sns.despine(fig=fig, ax=ax)
plt.xlabel('Date')
_ = plt.ylabel('Actual')

data_clean.groupby('Impact').size().plot(kind='barh', color=sns.palettes.mpl_palette('Dark2'))
plt.gca().spines[['top', 'right',]].set_visible(False)

# Splitting the data into features and target
X = data_clean.drop('Sentiment', axis=1)  # Features
y = data_clean['Sentiment']  # Target

# Engineering time-based features from Date column
X['DayOfWeek'] = pd.to_datetime(X['Date']).dt.dayofweek  # Extracting day of the week
X['Month'] = pd.to_datetime(X['Date']).dt.month  # Extracting month
X['Year'] = pd.to_datetime(X['Date']).dt.year  # Extracting year
X['Hour'] = pd.to_datetime(X['Time'], format='%H:%M').dt.hour #Extracting hour
X['Minute'] = pd.to_datetime(X['Time'], format='%H:%M').dt.minute #Extracting minute


# Drop unnecessary columns (Date, Description) for now
X = X.drop(['Date', 'Description'], axis=1)

# Splitting the data into train and test sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Preprocessing for numerical features
numeric_features = ['Actual', 'Forecast', 'Previous', 'DayOfWeek', 'Month', 'Hour', 'Minute']  # Include time-based features
numeric_transformer = StandardScaler()

# Preprocessing for text features (if Description is still used)
text_features = []  # Assuming Description column has been dropped
text_transformer = CountVectorizer()

# Combine preprocessing for numerical and text features
preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, numeric_features),
        ('text', text_transformer, text_features)  # Add text feature transformation here if needed
    ])

# Create the pipeline with preprocessing and model
pipeline_v2 = Pipeline(steps=[('preprocessor', preprocessor),
                           ('classifier', RandomForestClassifier())])  # You can change the classifier as needed

# Fit the model
pipeline_v2.fit(X_train, y_train)

# Predictions
predictions_v2 = pipeline_v2.predict(X_test)

# Evaluate the model
accuracy_pipeline_v2 = accuracy_score(y_test, predictions_v2)
report_pipeline_v2 = classification_report(y_test, predictions_v2, output_dict=True)

print(f"Accuracy: {accuracy_pipeline_v2}")
print(f"Classification Report:\n{report_pipeline_v2}")