# General
## Dates and Times
import pytz
from dateutil import parser
from datetime import timezone,datetime,timedelta
from dateutil.relativedelta import relativedelta

## String manipulation
import string
import re #regex
import json #json parsing
## Data Generation
import random

# Data Manipulation
import pandas as pd
import numpy as np
## Vector Math
from numpy.linalg import norm
## Math
import math

# Data Visualization
import matplotlib as mpl
import matplotlib.pyplot as plt
import plotly as pl
import plotly.express as px

# Data Science
import scipy as sp
import scipy.stats
import seaborn as sns
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.compose import ColumnTransformer
## Linear Regression
from sklearn.linear_model import LinearRegression
## ARMA
from statsmodels.graphics.tsaplots import plot_pacf
from statsmodels.graphics.tsaplots import plot_acf
from statsmodels.tsa.arima_process import ArmaProcess
from statsmodels.stats.diagnostic import acorr_ljungbox
from statsmodels.tsa.statespace.sarimax import SARIMAX
from statsmodels.tsa.stattools import adfuller
from statsmodels.tsa.stattools import pacf
from statsmodels.tsa.stattools import acf
## Nearest Neighbor
from sklearn.neighbors import NearestNeighbors
#import annoy #Aprox Nearest Neighbors
## NPL
#import spacy
#nlp = spacy.load('en_core_web_md')
#doc = nlp(open("txtfile.txt").read()) #Importing NPL txt file

# Scraper
import requests
from requests import get
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

df1 = pd.read_table('test.csv', sep=',')
df2 = pd.read_csv('test.csv')

print(df1)

df1.copy()

# Print Formatting
print(df1)

# Top Rows
df1.head()

# Column Names, Data Types
df1.columns

# Specific Data Types
df1.dtypes

# Row Count
df1.shape[0]
len(df1.index)

df1['col1']

# Row by Index
df1.loc[[1]]

# Row by Value
df1.loc[df1['col1'] == '1']

# Named Column by Row Index
df1['col1'].values[0]

# Range of Rows
df1[0:2]

# Range of Rows and Columns
df1[0:1][['col1','col2']]

# All column/row names
df3 = df1.copy()
df3.set_axis(['col_a', 'col_b', 'col_c', 'col_d', 'col_e'], axis=1, inplace=True) #axis=0 is x axis
print(df3.columns)

# Select column/row names
df4 = df1.rename(columns = {'col1':'col_a', 'col2':'col_b'})
print(df4.columns)

df3 = df3.set_index('col_a')
df3

df3 = df3.reset_index()
df3

pd.DataFrame(df3.set_index(['col_a','col_d']).stack())

# On Dataframe
df1.astype({
    'col4': 'string',
    'col5': 'int',
    'col1': 'int'
}, errors='ignore')

# On Column

df1['col4'].astype('string')

df4 = df4.drop(columns = ['col_a'])
df4

df4 = df1.rename(columns = {'col1':'col_a'})
df5 = df4[['col3', 'col2']].copy()
df5

# Example: Multiplication
df5 = df1.copy()
df5['new1'] = df5['col4'] * 10
df5['new2'] = df5['col5'] * 10
df5

# Example: Concatinate
df5['new3'] = df5['col1'] + '-' + df5['col2']
df5

# Example: Capitalizing String
str1 = df5['new1'].values[0]
print(str1.upper())

# Example: Trying to Capitalize Column (does not work)
df5['cap'] = df5['new1'].upper()

# Lambda Function
df5['cap'] = df5['new1'].apply(lambda x: x.upper())
df5

# Example: Filter by Column
df6 = df5.loc[df5['col5'] >= 12.5]
df6

# Example: Filter by Column
df6 = df5.loc[df5['new3'].str.contains(pat = r'a|B')]
df6

# Setup
d = {'col1': range(0,42,2), 'col2': range(10,220,10), 'col3': ('t','t','t','t','t','t','t','t','t','t','f','f','f','f','f','f','f','f','f','f','f')}
df2 = pd.DataFrame(data=d)
df2

# Aggregate Mean
df3 = df2.groupby(['col3'], axis=0).mean()
df3

# Aggregate Sum (without NaN values)
df3 = df2.groupby(['col3'], axis=0).sum().dropna()
df3

# Aggregate by different functions
df3 = df2.groupby(['col3'], axis=0).agg({'col1':'sum', 'col2':'mean'})
df3

d = {'col1': ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'), 'col2': range(0,7,1), 'col3': ('t','t','t','t','t','t','t')}
left = pd.DataFrame(data=d)

d = {'col1': ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'), 'col2': range(11,18,1), 'col3': ('f','f','f','f','f','f','f')}
right = pd.DataFrame(data=d)

# Full Merge
df1 = left.merge(right, how='inner', on='col1', suffixes=('_x','_y'), copy=False)
df1

# Merge Certain Columns
df2 = pd.merge(left, right[['col1', 'col2']], on='col1', suffixes=('_x','_y'))
df2

d = {'col1': ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'), 'col2': range(0,7,1), 'col3': ('t','t','t','t','t','t','t')}
left = pd.DataFrame(data=d)
left = left.set_index('col1')

d = {'col1': ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'), 'col2': range(11,18,1), 'col3': ('f','f','f','f','f','f','f')}
right = pd.DataFrame(data=d)
right = right.set_index('col1')

df1 = left.join(right, how='left', on='col1', lsuffix='_left', rsuffix='_right')
df1

d = {'col1': ('Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'), 'col2': range(0,7,1), 'col3': ('t','t','t','t','t','t','t')}
df1 = pd.DataFrame(data=d)

df1['Monday'] = np.where(df1['col1'] == 'Monday', 1, 0)

df1

d = {'col1': ('2023-01-01', '2023-01-02', '2023-01-03', '2023-01-04', '2023-01-05', '2023-01-06', '2023-01-07'), 'col2': range(0,7,1), 'col3': ('a','a','a','b','b','b','b')}
df1 = pd.DataFrame(data=d)
df1['col1'] = pd.to_datetime(df1['col1'])

# Shift by a period (date implied)
df1['col2_shift'] = df1.groupby(['col3'], as_index=False)['col2'].shift(periods=1)

# Cumsum function
df1[['col2_roll']] = df1.groupby(['col3'], as_index=False)['col2'].cumsum()
df1[['col2_roll2']] = df1[['col2']].cumsum()

# Window function
df1 = df1.merge(df1.groupby(['col3'], as_index=False)[['col1','col2']].rolling(2, on='col1', method='single').sum(), on=['col1','col3'], how='left')
df1 = df1.rename(columns={'col2_x':'col2','col2_y':'col2_window'})

df1

# Extract First Column (two ways)
ary1 = df5['col1'].to_numpy()
ary2 = np.array(df5['col1'])
ary1, ary2

# Extract All Columns
ary1 = np.array(df5.loc[0:, :])
ary1

# Extract First Row
ary1 = np.array(df5[0:1])
ary1

# Extract All Rows
ary1 = np.array(df5[0:])
ary1

# String Replacement
str1 = 'Hey Look A Dog'
str2 = str1.replace('Dog', 'Cat')
print(str2)

str3 = 'slash \\'
print(str3)
str3 = str3.replace('\\', '')
print(str3)

# String Contains Item
str1 = 'Hey Look A Dog'
str2 = 'Hey Look A Cat'
str3 = 'Dogs Are Cool'
str4 = 'Cats Are Cool'

print('Dog' in str1,',', 'Dog' in str2,',', 'dog' in str3,',', 'dog' in str4)

# String Contains df
d = {'col1': [1, 2, 3, 4], 'col2': [str1, str2, str3, str4]}
df2 = pd.DataFrame(data=d)
df2['tf'] = (df2['col2'].str.contains(pat = 'Dog'))
df2

# Regex
print(re.search(r'^D', str1),',', re.search(r'^D', str3),','
      , re.findall(r'^D', str1),',', re.findall(r'^D', str3))

# Pattern Finding
str1 = 'Thanksgiving this year is 11/24/2022'
str2 = re.findall('[0-9]{2}\/[0-9]{2}\/[0-9]{4}', str1)
print(str2[0])

# String Split
str1 = '01/01/2022'
l = str1.split('/')
print(l)

# Finding if Item in String List is in String
str1 = 'Where is the Store'
str2 = 'Hey Look A Cat'
str3 = 'Dogs Are Cool'
str4 = 'Who is That'
str5 = 'Frogs are the Best'
str6 = 'Llamas are Fun'
str7 = 'I like Clouds'
str8 = 'Bats are Awesome'
str9 = 'Dog Dog Dog'
str10 = 'Hello'

animals = ['Dog','Cat','Bat','Frog','Llama']

d = {'col1': range(1,11,1), 'col2': [str1, str2, str3, str4, str5, str6, str7, str8, str9, str10]}
df8 = pd.DataFrame(data=d)
df8['animal_tf'] = df8['col2'].apply(lambda x: any(y.upper() in x.upper() for y in animals))
df8['animal_listed'] = df8['col2'].apply(lambda x: animals[[y.upper() in x.upper() for y in animals].index(True)] if [y.upper() in x.upper() for y in animals].count(True) > 0 else '')
df8

# Vector Manipulation, Addinv Vectors to Vectors
v1 = np.array(['#1a2b3c', '#abcdef', '#000001'])
v2 = []
v3 = []

for i in v1:
    y = i.lstrip('#')
    v2.append(y)
    
    z = np.array([int(y[:2],16), int(y[2:4],16), int(y[4:6],16)])
    v3.append(z)

v2,v3

# Fill NaN
df1 = pd.DataFrame(np.nan, index=[0, 1, 2, 3], columns=['A', 'B'])
print(df1)
df2 = df1.fillna(value = '0.0')
print(df2)

# Numpy Vector Math
v1 = np.array([1,2,3])
v2 = np.array([2,3,4])

print(v1 * v2)
print(v2 / v1)

# Math functions
print(math.sqrt(v1[0]))

print(math.factorial(2))

print(math.fmod(19, 5))

# String to Datetime
datetime = pd.to_datetime('01/01/2022')
current_date = datetime.now() # current date

# Datetime Oject
'datetime64[ns]'

# Column to Date
pd.to_datetime(table['datetime_col'].apply(lambda x: x.date()))
pd.to_datetime(table['date'], format = '%b/%d/%Y')

# Time Delta (difference between two days)
numofdays(date2, date1)

c = date2 - date1
min_diff = c.seconds / 60 

# Extract Date Parts
datetime.year
datetime.month
datetime.day
datetime.date() # no time

# Format datetime
datetime.strftime('%Y-%m-%d') # YYYY-mm-dd format

# Add time to a date
next_year = current_date + relativedelta(years=1)

# ISO Week Month Function
def week_month(date):
    date = pd.to_datetime(date)
    year = date.strftime('%G')
    week = date.strftime('%V')
    start_date = pd.to_datetime(str(week) + '-Mon-' + str(year), format='%V-%a-%G')
    return start_date

# Corr Matrix
corr_matrix = ml_df.corr()
corr_matrix['y_col'].sort_values(ascending = False)

# Histograms
ml_df.hist(bins=50, figzise = (50,50))

# Seperate Y
ml_df_y = ml_df['y_col'].copy()

# Fill N/A
ml_df['column1'].fillna('None', inplace=True)

column2_med = ml_df['column2'].median 
ml_df['column2'].fillna(column2_med, inplace=True)

# Additional Columns
ml_df['extra_col'] = ml_df['column3'] / ml_df['column4']

# Drop Columns
ml_df = ml_df.drop(columns ={'bad_col'})

# Pipeline
cat_attribs = ['column1']
num_attribs = ['column2', 'column3', 'column4', 'extra_col']

num_pipeline = Pipeline([
    ('std_scaler', StandardScaler())
])

cat_pipeline = Pipeline([
    ('ohe', OneHotEncoder())
])

full_pipeline = ColumnTransformer([
    ('num', num_pipeline, num_attribs),
    ('cat', cat_pipeline, cat_attribs),
    
])

ml_df_matrix = full_pipeline.fit_transform(ml_df)

# Split

train_set_x, test_set_x, train_set_y, test_set_y = train_test_split(ml_df_matrix, ml_df_y, test_size=0.2, random_state=3)

# Feature Names
pipleline = make_pipeline(full_pipeline, LinearRegression())
pipeline.fit(ml_df, ml_df['y_col'])
feature_names = pipeline[:-1].get_feature_names_out()

test_set_y = np.array(test_set_y, dtype=float)
train_set_y = np.array(train_set_y, dtype=float)

train_set_mean = train_set_y.mean()
avg_array = [train_set_mean] * len(test_set_y)

avg_mse = mean_squared_error(test_set_y, avg_array)
avg_rmse = np.sqrt(avg_mse)

# Train
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(train_set_x, train_set_y)

# Evaluate
test_set_y = np.array(test_set_y, dtype=float)
predict = forest_reg.predict(test_set_y)
forest_mse = mean_squared_error(test_set_y, predict)