MSCI.TW30.Pearson correlation coefficient,Draft,發散的亂算

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import pearsonr

# Mount Google Drive
from google.colab import drive
drive.mount('/content/drive', force_remount=True)

# Load the data
file_path = '/content/drive/My Drive/MSCI_Taiwan_30_data.csv'
data = pd.read_csv(file_path)

# Select only numerical columns
numerical_data = data.select_dtypes(include=[np.number])

# Handle NaN and infinite values by replacing them with the mean of the column
numerical_data = numerical_data.apply(lambda x: np.where(np.isfinite(x), x, np.nan))
numerical_data = numerical_data.apply(lambda x: x.fillna(x.mean()), axis=0)

# Calculate Pearson correlation with target variable 'Close_TAIEX'
target_variable = 'Close_TAIEX'
correlation_results = {}

for column in numerical_data.columns:
    if column != target_variable:
        correlation, p_value = pearsonr(numerical_data[target_variable], numerical_data[column])
        correlation_results[column] = {'Pearson Correlation': correlation, 'P-value': p_value}

# Convert results to DataFrame for better visualization
correlation_df = pd.DataFrame.from_dict(correlation_results, orient='index')
correlation_df = correlation_df.sort_values(by='Pearson Correlation', ascending=False)

print("Pearson Correlation with Close_TAIEX and corresponding P-values:")
print(correlation_df)

# Plot scatter plots for each explanatory variable vs. Close_TAIEX
plt.figure(figsize=(20, 20))
for i, column in enumerate(correlation_df.index):
    plt.subplot(6, 6, i + 1)  # Adjust the number of rows and columns based on the number of variables
    sns.scatterplot(x=numerical_data[target_variable], y=numerical_data[column])
    plt.title(f'{column} vs. {target_variable}\n(r={correlation_df.loc[column, "Pearson Correlation"]:.2f}, p={correlation_df.loc[column, "P-value"]:.2e})')
    plt.xlabel(target_variable)
    plt.ylabel(column)
plt.tight_layout()
plt.show()

# Import necessary libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# Mount Google Drive
from google.colab import drive
drive.mount('/content/drive')

# Load the data
file_path = '/content/drive/My Drive/MSCI_Taiwan_30_data.csv'
data = pd.read_csv(file_path)

# Descriptive statistics
desc_stats = data.describe()
print("Descriptive Statistics:")
print(desc_stats)

# Box plot to visualize distribution and check for outliers
plt.figure(figsize=(16, 10))
sns.boxplot(data=data.select_dtypes(include=[np.number]))  # Only plot numerical columns
plt.xticks(rotation=90)
plt.title('Box Plot of Features')
plt.show()

# Pearson correlation coefficient
numerical_data = data.select_dtypes(include=[np.number])  # Select only numerical columns
correlation_matrix = numerical_data.corr(method='pearson')

# Heatmap to visualize correlation between variables
plt.figure(figsize=(16, 10))
sns.heatmap(correlation_matrix, annot=True, fmt='.2f', cmap='coolwarm')
plt.title('Heatmap of Pearson Correlation Coefficients')
plt.show()