import tensorflow as tf
import numpy as np

print(tf.__version__)

# x_data, y_data 정의
x_data = [1, 2, 3, 4, 5]
y_data = [1, 2, 3, 4, 5]

# matplot import
import matplotlib.pyplot as plt
plt.plot(x_data, y_data, 'o')
plt.ylim(0, 8)

# reduce_mean의 차원감소 평균 사용
v =[1., 2., 3., 4.]
tf.reduce_mean(v) # 2.5

# 3의 제곱근
tf.square(3) # 9

# x_data, y_data 정의
x_data = [1, 2, 3, 4, 5]
y_data = [1, 2, 3, 4, 5]

# Weight Bias 정의
W = tf.Variable(2.0)
b = tf.Variable(0.5)

# 가설 정의
hypothesis = W * x_data + b

# Weight Bias Numpy 출력
W.numpy(), b.numpy()

# Hypothesis Numpy 출력
hypothesis.numpy()

# x_data, hypothesis -> R
# x_data, y_data -> O
plt.plot(x_data, hypothesis.numpy(), 'r-')
plt.plot(x_data, y_data, 'o')
plt.ylim(0, 12)
plt.show()

# Cost 함수 정의 Average((H-Y)^2)
cost = tf.reduce_mean(tf.square(hypothesis - y_data))

# 경사 하강법 Tape 정의
with tf.GradientTape() as tape:
    hypothesis = W * x_data + b
    cost = tf.reduce_mean(tf.square(hypothesis - y_data))

# Weight, Bias
W_grad, b_grad = tape.gradient(cost, [W, b])
W_grad.numpy(), b_grad.numpy()

# 학습율
learning_rate = 0.01

# Weight - (Weight * Learning Rate)
W.assign_sub(learning_rate * W_grad)

# Bias - (Bias * Learning Rate)
b.assign_sub(learning_rate * b_grad)

W.numpy(), b.numpy()

plt.plot(x_data, hypothesis.numpy(), 'r-')
plt.plot(x_data, y_data, 'o')
plt.ylim(0, 12)

# Weight / Bias 정의
W = tf.Variable(2.9)
b = tf.Variable(0.5)

# for loop 수행
for i in range(100):
    # Gradient Tap e정의
    with tf.GradientTape() as tape:
        hypothesis = W * x_data + b
        cost = tf.reduce_mean(tf.square(hypothesis - y_data))

    # Weight Bias 정의
    W_grad, b_grad = tape.gradient(cost, [W, b])

    # Weight - (Learning Rate * Weight)
    W.assign_sub(learning_rate * W_grad)
    # Bias - (Learning Rate * Bias)
    b.assign_sub(learning_rate * b_grad)

    # Step 10에서 출력
    if i % 10 == 0:
      print("{:5}|{:10.4f}|{:10.4f}|{:10.6f}".format(i, W.numpy(), b.numpy(), cost))
      
    # x_data, y_data 원본
    plt.plot(x_data, y_data, 'o')
    # x_data, hypothesis 가설
    plt.plot(x_data, hypothesis.numpy(), 'r-')
    plt.ylim(0, 18)

print(W * 5 + b)
print(W * 2.5 + b)

# import library
import tensorflow as tf
import numpy as np

# tensor flow version 확인
print(tf.__version__)

# X,Y 1,2,3 np array 선언 
X = np.array([1, 2, 3])
Y = np.array([1, 2, 3])

# cost_func 정의 (Python형태)
# Weight, X, Y를 인자로 전달
def cost_func_py(W, X, Y):
    # c 를 0으로 초기화
    c = 0
    # X 만큼 Loop를 돌며 (W*X - Y) ^ 2 -> C로 대입
    # C (오차 제곱근)을 X로 평균낸 값을 return
    for i in range(len(X)):
        c += (W * X[i] - Y[i]) ** 2
    return c / len(X)

# cost_func 정의 (tensorflow형태)
# Weight, X, Y를 인자로 전달
def cost_func_tf(W, X, Y):
  hypothesis = X * W
  return tf.reduce_mean(tf.square(hypothesis - Y))

# Weight를 -3부터 5까지 9개의 요소로 제공 
W_values = np.linspace(-10, 12, num=23)
cost_values = []

# numpy.linspace (start,end,number)
# 제공된 Weight, X, Y로 Cost를 계산
for feed_W in W_values:
    curr_cost_py = cost_func_py(feed_W, X, Y)    
    curr_cost_tf = cost_func_tf(feed_W, X, Y)
    cost_values.append(curr_cost_tf)
    print("{:6.3f} | {:10.5f} | {:10.5f}".format(feed_W, curr_cost_py, curr_cost_tf))

# matplot import
import matplotlib.pyplot as plt

# figure size 지정
plt.rcParams["figure.figsize"] = (8,6)
plt.plot(W_values, cost_values, "b")
plt.ylabel('Cost(W)')
plt.xlabel('W')
plt.show()

##################################################################
# random 사용시 Seed 설정
tf.random.set_seed(0)  # for reproducibility

x_data = [1., 2., 3., 4.]
y_data = [1., 3., 5., 7.]

# random.normal((배열),mean,stddev)
W = tf.Variable(tf.random.normal((1,), -100., 100.))

# 300까지 for 
for step in range(300):
    # 가설을 W*X로 정의
    hypothesis = W * X
    # TF의 Cost 함수 사용
    cost = tf.reduce_mean(tf.square(hypothesis - Y))

    # Learning Rate 0.01
    alpha = 0.01

    # 경사 하강법 : 1/m * (W*X - Y) * X
    gradient = tf.reduce_mean(tf.multiply(tf.multiply(W, X) - Y, X))

    # Weight - Learning Rate * Weight
    descent = W - tf.multiply(alpha, gradient)

    # Assing New Weight
    W.assign(descent)
    
    # 10 step 마다 로그
    if step % 10 == 0:
        print('{:5} | {:15.6f} | {:10.6f}'.format( step, cost.numpy(), W.numpy()[0]) )

print(5.0 * W)
print(2.5 * W)

##################################################################
x_data = [1., 2., 3., 4.]
y_data = [1., 3., 5., 7.]

# Weight 5부터 접근
W = tf.Variable([5.0])

# 300까지 for 
for step in range(300):
    # 가설을 W*X로 정의
    hypothesis = W * X
    # TF의 Cost 함수 사용
    cost = tf.reduce_mean(tf.square(hypothesis - Y))

    # Learning Rate 0.01
    alpha = 0.01

    # 경사 하강법 : 1/m * (W*X - Y) * X
    gradient = tf.reduce_mean(tf.multiply(tf.multiply(W, X) - Y, X))

    # Weight - Learning Rate * Weight
    descent = W - tf.multiply(alpha, gradient)

    # Assing New Weight
    W.assign(descent)
    
    # 10 step 마다 로그
    if step % 10 == 0:
        print('{:5} | {:10.4f} | {:10.6f}'.format(step, cost.numpy(), W.numpy()[0]))

# LIbrary 선언
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf

tf.random.set_seed(777)  # for reproducibility
print(tf.__version__)

# x_train data, y_train data 정의
x_train = [[1., 2.],
           [2., 3.],
           [3., 1.],
           [4., 3.],
           [5., 3.],
           [6., 2.]]
y_train = [[0.],
           [0.],
           [1.],
           [1.],
           [1.],
           [1.]]

# x_test data, y_test data
x_test = [[2.,2.]]
y_test = [[0.]]


# x_train data의 x의 1열이 x1
# x_train data의 x의 2열이 x2
x1 = [x[0] for x in x_train]
x2 = [x[1] for x in x_train]

# y_train data의 y의 0열의 3나누기 나머지
colors = [int(y[0] % 3) for y in y_train]

# x_train data scatter plot
plt.scatter(x1,x2, c=colors , marker='x')

# x_test data scatter plot
# Test 데이터는 붉은색의 위치와 같이 추론시 1의 값을 가지게 됩니다.
plt.scatter(x_test[0][0],x_test[0][1], c="red")

plt.xlabel("x1")
plt.ylabel("x2")
plt.show()

# 학습 dataset 정의 
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(len(x_train))#.repeat()

# Weight 2*1 행렬의 0값으로 초기화
# Bias 1 벡터의 0값으로 초기화
W = tf.Variable(tf.zeros([2,1]), name='weight')
b = tf.Variable(tf.zeros([1]), name='bias')

# hypothesis = ( 1 / (1 + exponential ((fetures * W) + b)) ) <- Sigmoid
# https://upload.wikimedia.org/wikipedia/commons/8/88/Logistic-curve.svg
def logistic_regression(features):
    hypothesis  = tf.divide(1., 1. + tf.exp(tf.matmul(features, W) + b))
    return hypothesis

# 손실 함수 정의   - ( y * log( hypothesis(x) ) + (1- y) log(1-hypothesis) )
def loss_fn(hypothesis, features, labels):
    cost = -tf.reduce_mean(labels * tf.math.log(logistic_regression(features)) + (1 - labels) * tf.math.log(1 - hypothesis))
    return cost

# Activation SGD로 설정 
# Learning Rate 0.01로 설정
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)

# Accuracy Function은 hypothesis, labes(Y)를 인자로 받음
# predicted = Type Cast (Hypothesis > 0.5 일 경우 Float 32)
# accuracy = predicted, labels를 비교하여 boolean을 반납후 int 32로 cast한 값의 평균을 구함
def accuracy_fn(hypothesis, labels):
    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.int32))
    return accuracy

# Gradient Tape을 통해 경사값을 계산
def grad(features, labels):
    with tf.GradientTape() as tape:
        # 손실 값은 loss_fn(logistic_regression(X), X, Y) 으로 정의
        # loss_fn은 Sigmoid
        loss_value = loss_fn(logistic_regression(features),features,labels)
    return tape.gradient(loss_value, [W,b])

# 1000회 반복 수행
EPOCHS = 1001

# for loop 1000회
for step in range(EPOCHS):
    for features, labels  in iter(dataset):
        # Grad (X, Y)
        grads = grad(features, labels)
        # SGD Optimizer Vriable 설정
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))
        # 100 Step 마다 loss log
        if step % 100 == 0:
            print("Iter: {}, Loss: {:.4f}".format(step, loss_fn(logistic_regression(features),features,labels)))

# Accuracy Function에 Y', Y를 대입하고 Test Accuracy 산출후 출력
test_acc = accuracy_fn(logistic_regression(x_test),y_test)
print("Testset Accuracy: {:.4f}".format(test_acc))  

x_test = [[5.,2.]]
y_test = [[1.]]
plt.scatter(x_test[0][0],x_test[0][1], c="red")
plt.xlabel("x1")
plt.ylabel("x2")
plt.show()

# Accuracy Function에 Y', Y를 대입하고 Test Accuracy 산출후 출력
test_acc = accuracy_fn(logistic_regression(x_test),y_test)
print("Testset Accuracy: {:.4f}".format(test_acc))
print("Y: ",format(y_test),format(logistic_regression(x_test)) )

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf

tf.random.set_seed(777)  # for reproducibility
print(tf.__version__)

# Google Driver 사용을 위한 Mount
from google.colab import drive
drive.mount('/content/drive')

# csv data load
xy = np.loadtxt('/content/drive/My Drive/Colab Notebooks/dl4all_2/data-03-diabetes.csv', delimiter=',', dtype=np.float32)
# x_train 전체행 1열부터 마지막 전열
x_train = xy[:, 0:-1]
# y_train 전체행 마지막 열
y_train = xy[:, [-1]]

# shape 출력
print(x_train.shape, y_train.shape)

# xy 출력
print(xy)

dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(len(x_train))

# Weight와 Bias를 Random으로 설정
W = tf.Variable(tf.random.normal((8, 1)), name='weight')
b = tf.Variable(tf.random.normal((1,)), name='bias')

# Activation : Hypothesis = 1 / ( 1 + exponential ( X matmul W ) + b )
def logistic_regression(features):
    hypothesis  = tf.divide(1., 1. + tf.exp(tf.matmul(features, W) + b))
    return hypothesis

# 손실 함수 정의   - ( y * log( hypothesis(x) ) + (1- y) log(1-hypothesis) )
def loss_fn(hypothesis, features, labels):
    cost = -tf.reduce_mean(labels * tf.math.log(logistic_regression(features)) + (1 - labels) * tf.math.log(1 - hypothesis))
    return cost

# Activation SGD 
# Learning Rate 0.01
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)

# Sigmoid 함수를 통해 예측값이 0.5보다 크면 1을 반환하고 0.5보다 작으면 0으로 반환
def accuracy_fn(hypothesis, labels):
    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.int32))
    return accuracy

# GradientTape을 통해 경사값을 계산
def grad(hypothesis, features, labels):
    with tf.GradientTape() as tape:
        loss_value = loss_fn(logistic_regression(features),features,labels)
    return tape.gradient(loss_value, [W,b])

# 반복횟수 1000회
EPOCHS = 1001

# Epochs 만큼 for loop
for step in range(EPOCHS):
    # dataset에 fetures와 labels 만큼 for loop
    for features, labels  in iter(dataset):
        # gradients 정의
        grads = grad(logistic_regression(features), features, labels)

        # SGD Optimizer Variable Setting
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))

        # 100 Step 마다 로그
        if step % 100 == 0:
            print("Iter: {}, Loss: {:.4f}".format(step, loss_fn(logistic_regression(features),features,labels)))

import tensorflow as tf
import numpy as np
print(tf.__version__)
tf.random.set_seed(777)  # for reproducibility

# x_data, y_data 정의
x_data = [[1, 2, 1, 1],
          [2, 1, 3, 2],
          [3, 1, 3, 4],
          [4, 1, 5, 5],
          [1, 7, 5, 5],
          [1, 2, 5, 6],
          [1, 6, 6, 6],
          [1, 7, 7, 7]]
y_data = [[0, 0, 1],
          [0, 0, 1],
          [0, 0, 1],
          [0, 1, 0],
          [0, 1, 0],
          [0, 1, 0],
          [1, 0, 0],
          [1, 0, 0]]

#convert into numpy and float format
# numpy array float32 data로 converting
x_data = np.asarray(x_data, dtype=np.float32)
y_data = np.asarray(y_data, dtype=np.float32)

# dataset을 선언합니다.
dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))
dataset = dataset.repeat().batch(2)
nb_classes = 3 #class의 개수입니다.

print(x_data.shape)
print(y_data.shape)

#Weight and bias setting
# Weight random.normal (4,3)
# Bias random.normal (3)
W = tf.Variable(tf.random.normal((4, nb_classes)), name='weight')
b = tf.Variable(tf.random.normal((nb_classes,)), name='bias')
variables = [W, b]

print(W,b)

# tf.nn.softmax computes softmax activations
# softmax = exp(logits) / reduce_sum(exp(logits), dim)
# softmax softmax ( X * W + B )
def hypothesis(X):
    return tf.nn.softmax(tf.matmul(X, W) + b)

# h(x) print
print(hypothesis(x_data))

# Softmax onehot test
# Softmax 테스트
sample_db = [[8,2,1,4]]
sample_db = np.asarray(sample_db, dtype=np.float32)

print(hypothesis(sample_db))

# Cost Function
def cost_fn(X, Y):
    # H(X) = Logit
    logits = hypothesis(X)

    # Cost =  SUM ( Y * log(H(X)) ) 
    cost = -tf.reduce_sum(Y * tf.math.log(logits), axis=1)

    # cost_mean =  mean( cost )
    cost_mean = tf.reduce_mean(cost)
    
    return cost_mean

print(cost_fn(x_data, y_data))

# x = 3
x = tf.constant(3.0)

# GradientTape 정의
with tf.GradientTape() as g:
    # g.watch : Ensures that tensor is being traced by this tape.
    g.watch(x)
    y = x * x # x^2

# delta y , delta x
dy_dx = g.gradient(y, x) # Will compute to 6.0
print(dy_dx)

# Gradient Funtion
def grad_fn(X, Y):
    with tf.GradientTape() as tape:
        # loss = cost_mean
        loss = cost_fn(X, Y)

        # gradient ( loss, )
        grads = tape.gradient(loss, variables)

        return grads

print(grad_fn(x_data, y_data))

# fitting : X,Y, 2000 epochs, verbose 100
def fit(X, Y, epochs=2000, verbose=100):
    # learning rate setting
    optimizer =  tf.keras.optimizers.SGD(learning_rate=0.1)

    # for loop 
    for i in range(epochs):
        # gradient function X, Y
        grads = grad_fn(X, Y)

        # Optimizer Variable Seting
        optimizer.apply_gradients(zip(grads, variables))

        # verbose 100 step log
        if (i==0) | ((i+1)%verbose==0):
            print('Loss at epoch %d: %f' %(i+1, cost_fn(X, Y).numpy()))

# start fitting loop
fit(x_data, y_data)

# Sample 2, 1, 3, 2 -> 0,0,1
sample_data = [[2,1,3,2]] # answer_label [[0,0,1]]
sample_data = np.asarray(sample_data, dtype=np.float32)

a = hypothesis(sample_data)

print(a)
print(tf.argmax(a, 1)) #index: 2

sample_data = [[1,6,4,4]] 
sample_data = np.asarray(sample_data, dtype=np.float32)

a = hypothesis(sample_data)

print(a)
print(tf.argmax(a, 1)) #index: 2

b = hypothesis(x_data)
print(b)
print(tf.argmax(b, 1))
print(tf.argmax(y_data, 1)) # matches with y_data

# softmax classifer
class softmax_classifer(tf.keras.Model):
    # class initilization
    def __init__(self, nb_classes):
        super(softmax_classifer, self).__init__()
        self.W = tf.Variable(tf.random.normal((4, nb_classes)), name='weight')
        self.b = tf.Variable(tf.random.normal((nb_classes,)), name='bias')
        
    # softmax regression
    def softmax_regression(self, X):
        return tf.nn.softmax(tf.matmul(X, self.W) + self.b)
    
    # cost_function
    def cost_fn(self, X, Y):
        logits = self.softmax_regression(X)
        cost = tf.reduce_mean(-tf.reduce_sum(Y * tf.math.log(logits), axis=1))        
        return cost
    
    # gradient function
    def grad_fn(self, X, Y):
        with tf.GradientTape() as tape:
            cost = self.cost_fn(x_data, y_data)
            grads = tape.gradient(cost, self.variables)            
            return grads
    
    # fitting
    def fit(self, X, Y, epochs=2000, verbose=500):
        optimizer =  tf.keras.optimizers.SGD(learning_rate=0.1)

        for i in range(epochs):
            grads = self.grad_fn(X, Y)
            optimizer.apply_gradients(zip(grads, self.variables))
            if (i==0) | ((i+1)%verbose==0):
                print('Loss at epoch %d: %f' %(i+1, self.cost_fn(X, Y).numpy()))
            
model = softmax_classifer(nb_classes)
model.fit(x_data, y_data)

import tensorflow as tf
import numpy as np
print(tf.__version__)
tf.random.set_seed(777)  # for reproducibility

# Google Driver 사용을 위한 Mount
from google.colab import drive
drive.mount('/content/drive')

# csv data load
xy = np.loadtxt('/content/drive/My Drive/Colab Notebooks/dl4all_2/data-04-zoo.csv', delimiter=',', dtype=np.float32)
# x_data 전체행 1열부터 마지막 전열
x_data = xy[:, 0:-1]
# y_data 전체행 마지막열
y_data = xy[:, -1]

# nb_classes 7 
nb_classes = 7  # 0 ~ 6

# Make Y data as onehot shape
# Y Data One-Hot Encoding
Y_one_hot = tf.one_hot(y_data.astype(np.int32), nb_classes)

print(x_data.shape, Y_one_hot.shape, y_data.shape, Y_one_hot, y_data)

# Weight and bias setting
# Weight = 16, 7
# Bias = 16
W = tf.Variable(tf.random.normal((16, nb_classes)), name='weight')
b = tf.Variable(tf.random.normal((nb_classes,)), name='bias')
variables = [W, b]

# tf.nn.softmax computes softmax activations
# softmax = exp(logits) / reduce_sum(exp(logits), dim)
# Logit Function = X * W + B
# cross_entropy_with_logist에서 logit_fn을 받음
def logit_fn(X):
    return tf.matmul(X, W) + b

# Hypothesis = Softmax ( Logit Function )
# prediction에서 hypothesis 받음
def hypothesis(X):
    return tf.nn.softmax(logit_fn(X))

# Cost Function 
# Logits = Logit Function
# Cost_i = Cross Entropy ( Y , Logit )
# Cost Reduce Mean ( Cost_i )
def cost_fn(X, Y):
    logits = logit_fn(X)
    cost_i = tf.keras.losses.categorical_crossentropy(y_true=Y, y_pred=logits, from_logits=True)    
    cost = tf.reduce_mean(cost_i)    
    return cost

# Gradient Tape
def grad_fn(X, Y):
    with tf.GradientTape() as tape:
        # Cost Function ( X, Y )
        loss = cost_fn(X, Y)
        grads = tape.gradient(loss, variables)
        return grads

# Prediction : ArgMax (Hypothesis(X), 1)
# Correct Prediction
# Accuracy = Average ( Prediction Float 32 Type Casting )
def prediction(X, Y):
    # Argument 중 Max값을 Prediction Value로 선택
    pred = tf.argmax(hypothesis(X), 1)
    # Prediction과 Argument Max의 동일 여부를 선택
    correct_prediction = tf.equal(pred, tf.argmax(Y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    return accuracy

# fitting function
def fit(X, Y, epochs=1000, verbose=100):
    # SGD Optimizer : Learning Rate 0.1
    optimizer =  tf.keras.optimizers.SGD(learning_rate=0.1)
e
    # Epochs for loop
    for i in range(epochs):
        grads = grad_fn(X, Y)
        optimizer.apply_gradients(zip(grads, variables))

        if (i==0) | ((i+1)%verbose==0):
#             print('Loss at epoch %d: %f' %(i+1, cost_fn(X, Y).numpy()))
            acc = prediction(X, Y).numpy()
            loss = cost_fn(X, Y).numpy() 
            print('Steps: {} Loss: {}, Acc: {}'.format(i+1, loss, acc))

fit(x_data, Y_one_hot)

# import
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf
from mpl_toolkits.mplot3d import Axes3D

tf.random.set_seed(777)  # for reproducibility

print(tf.__version__)

# x_train data
x_train = [[1, 2, 1],
          [1, 3, 2],
          [1, 3, 4],
          [1, 5, 5],
          [1, 7, 5],
          [1, 2, 5],
          [1, 6, 6],
          [1, 7, 7]]

# y_train data
y_train = [[0, 0, 1],
          [0, 0, 1],
          [0, 0, 1],
          [0, 1, 0],
          [0, 1, 0],
          [0, 1, 0],
          [1, 0, 0],
          [1, 0, 0]]

# Evaluation our model using this test dataset
# x_test data
x_test = [[2, 1, 1],
          [3, 1, 2],
          [3, 3, 4]]
# y_test data
y_test = [[0, 0, 1],
          [0, 0, 1],
          [0, 0, 1]]

# x_train data
x1 = [x[0] for x in x_train]
x2 = [x[1] for x in x_train]
x3 = [x[2] for x in x_train]

# plot figure 3d
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x1, x2, x3, c=y_train, marker='^')

# test data 
ax.scatter(x_test[0][0], x_test[0][1], x_test[0][2], c="red", marker='x')
ax.scatter(x_test[1][0], x_test[1][1], x_test[1][2], c="red", marker='x')
ax.scatter(x_test[2][0], x_test[2][1], x_test[2][2], c="red", marker='x')

ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')

plt.show()

# Tensorflow data API를 통해 학습시킬 값을 담는다 (Batch Size는 한번에 학습시킬 Size로 정함)
# dataset 정의
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(len(x_train))#.repeat()

# Weight Bias Random 정의
W = tf.Variable(tf.random.normal((3, 3)))
b = tf.Variable(tf.random.normal((3,)))

# Softmax를 가설로 선언 : Softmax를 통해 가장 높은 값을 구함
# softmax_fn 선언 (Hypothesis = Softmax( F(x)를 Return))
def softmax_fn(features):
    # hypothesis = softmax ( fetures (x) * Weight + Bias )
    hypothesis = tf.nn.softmax(tf.matmul(features, W) + b)
    return hypothesis

# 가설을 검증할 Cost 함수를 정의
# Average ( - SUM ( Labels (Y) * Log (Softmax(H(X))) ) )
def loss_fn(hypothesis, features, labels):
    cost = tf.reduce_mean(-tf.reduce_sum(labels * tf.math.log(hypothesis), axis=1))
    return cost

# Boolean is_decay 설정
is_decay = True

# Start Learning Rate 0.1 설정
starter_learning_rate = 0.1

# Decaying the learning rate
# tf.train.exponential_decay : This function applies an exponential decay function to a provided initial learning rate
# tf.train.inverse_time_decay : It is often recommended to lower the learning rate as the training progresses
# tf.train.natural_exp_decay : This function applies an exponential decay function to a provided initial learning rate
# tf.train.piecewise_constant : Piecewise constant from boundaries and interval values.
# tf.train.polynomial_decay : It is commonly observed that a monotonically decreasing learning rate, whose degree of change is carefully chosen, results in a better performing model.
# tf.train.cosine_decay : When training a model, it is often recommended to lower the learning rate as the training progresses.
# tf.train.linear_cosine_decay : Note that linear cosine decay is more aggressive than cosine decay and larger initial learning rates can typically be used.
# tf.train.noisy_linear_cosine_decay : Note that linear cosine decay is more aggressive than cosine decay and larger initial learning rates can typically be used.

# is_decay true 일경우
if(is_decay):    
    # Learning Rate = Exponential Deacy ( 시작 LR, Step, Rate, Staricase 여부 )
    # exp_rate = starter_rate * exp ( decay_rate * t )
    # initial_learning_rate : The inital Learning Rate
    # global_step : 현재 학습 횟수, Global step to use for the decay computation. Must not be negative.
    # decay_steps : 총학습 횟수, Must be positive
    # decay_rate : 감소 비율, * decay_rate
    # staricase : 이산적 학습 속도 감속 유무 ( global_step / decay_steps ) 의 제곱 적용 여부
    # decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
    learning_rate = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=starter_learning_rate,
                                                                 decay_steps=1000,
                                                                 decay_rate=0.96,
                                                                 staircase=True)
    # SGD Optimizer
    optimizer = tf.keras.optimizers.SGD(learning_rate)
else:
    # is_decay가 false면 초기 Learning Rate 적용
    optimizer = tf.keras.optimizers.SGD(learning_rate=starter_learning_rate)

# Gradient Tape
def grad(hypothesis, features, labels):
    with tf.GradientTape() as tape:
        # Loss Value = Loss Function ( softmax (X) , x , y )
        loss_value = loss_fn(softmax_fn(features),features,labels)
    # Gradient Tape ( cost , [w, b] )
    return tape.gradient(loss_value, [W,b])

# Accuracy Function
def accuracy_fn(hypothesis, labels):
    # Prediction = Hypothesis중 Max
    prediction = tf.argmax(hypothesis, 1)
    # Equal 판정
    is_correct = tf.equal(prediction, tf.argmax(labels, 1))
    # 정확도 측정
    accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
    # Accuracy Return
    return accuracy

# epochs 1000
EPOCHS = 1001

# epoch 만큼 for loop
for step in range(EPOCHS):
    # dataset 의 x, y 만큼 for loop
    for features, labels  in iter(dataset):
        # features (x)  type cast
        features = tf.cast(features, tf.float32)
        # labes (y) type cast
        labels = tf.cast(labels, tf.float32)
        # Gradient Tape
        grads = grad(softmax_fn(features), features, labels)
        # Optimizer
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))
        # verbose 100 
        if step % 100 == 0:
            print("Iter: {}, Loss: {:.4f}".format(step, loss_fn(softmax_fn(features),features,labels)))

# Test Set 수행
x_test = tf.cast(x_test, tf.float32)
y_test = tf.cast(y_test, tf.float32)
test_acc = accuracy_fn(softmax_fn(x_test),y_test)
print("Testset Accuracy: {:.4f}".format(test_acc))

# import
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf

tf.random.set_seed(777)  # for reproducibility

print(tf.__version__)

# xy 정의
xy = np.array([[828.659973, 833.450012, 908100, 828.349976, 831.659973],
               [823.02002, 828.070007, 1828100, 821.655029, 828.070007],
               [819.929993, 824.400024, 1438100, 818.97998, 824.159973],
               [816, 820.958984, 1008100, 815.48999, 819.23999],
               [819.359985, 823, 1188100, 818.469971, 818.97998],
               [819, 823, 1198100, 816, 820.450012],
               [811.700012, 815.25, 1098100, 809.780029, 813.669983],
               [809.51001, 816.659973, 1398100, 804.539978, 809.559998]])

# x_train 전행, 1열부터 마지막 전열
# y_train 전행, 마지막열
x_train = xy[:, 0:-1]
y_train = xy[:, [-1]]

# 이상치에 의해 값이 왜곡됨
plt.plot(x_train, 'ro')
plt.plot(y_train)
plt.show()

# Tensorflow data API를 통해 학습시킬 값을 담음 (Batch Size는 한번에 학습시킬 Size로 정함)
# X(features),Y(labels)는 실재 학습에 쓰일 Data (연산을 위해 Type를 맞춰줌)
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(len(x_train))

# Weight, Bias Random 초기화
W = tf.Variable(tf.random.normal((4, 1)), dtype=tf.float32)
b = tf.Variable(tf.random.normal((1,)), dtype=tf.float32)

# Linear Regression Hypothesis 정의 : X * Weight + Bias
def linearReg_fn(features):
    hypothesis = tf.matmul(features, W) + b
    return hypothesis

# Loss Function 정의 : Average ( ( F(x) - Y ) ^ 2 )
def loss_fn(hypothesis, features, labels):
    cost = tf.reduce_mean(tf.square(hypothesis - labels))
    return cost

# SGD Optimizer 선택 : Learning Rate 0.00005
optimizer = tf.keras.optimizers.SGD(learning_rate=1e-5)

# Gradient Tape 
def grad(hypothesis, features, labels):
    with tf.GradientTape() as tape:
        # Loss Value = Loss Function ( LinearRegression ( x ) , x, y )
        loss_value = loss_fn(linearReg_fn(features),features,labels)
    return tape.gradient(loss_value, [W,b]), loss_value

# Epochs 100회
EPOCHS = 101

# For loop 
for step in range(EPOCHS):
    # Dataset X, Y
    for features, labels  in dataset:
        # X 의 Type Cast
        features = tf.cast(features, tf.float32)

        # Y 의 Type Case
        labels = tf.cast(labels, tf.float32)

        # Hypothesis Value = Linear Regression Function ( X )
        hypo_value = linearReg_fn(features)

        # GradientDecenst Value
        grads, loss_value = grad(linearReg_fn(features), features, labels)        
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))    
    # Log without Verbose
    print("Iter: {}, Loss: {:.4f}, Prediction: {}".format(step, loss_value, hypo_value))

# xy 정의
xy = np.array([[828.659973, 833.450012,  908100, 828.349976, 831.659973],
               [823.02002,  828.070007, 1828100, 821.655029, 828.070007],
               [819.929993, 824.400024, 1438100, 818.97998,  824.159973],
               [816,        820.958984, 1008100, 815.48999,  819.23999],
               [819.359985, 823,        1188100, 818.469971, 818.97998],
               [819,        823,        1198100, 816,        820.450012],
               [811.700012, 815.25,     1098100, 809.780029, 813.669983],
               [809.51001,  816.659973, 1398100, 804.539978, 809.559998]])

# Normalization 정의 
def normalization(data):
    # data 를 인자로 받음
    # numerator = 개별 data - 열의 min값 
    numerator = data - np.min(data, 0)
    # denominator = 열의 Max값 - 열의 Min값
    denominator = np.max(data, 0) - np.min(data, 0)
    # print("data : ", data, "np_min : ", np.min(data,0), "np_max : ", np.max(data,0),"num : ", numerator, "den : ", denominator)
    # 열의 값을 Min 0 ~ Max 1의 비율로 Normalization
    return numerator / denominator

print(xy)

xy = normalization(xy)

print(xy)

x_train = xy[:, 0:-1]
y_train = xy[:, [-1]]

plt.plot(x_train, 'ro')
plt.plot(y_train)

plt.show()

# dataset, Weight, Bias 정의
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).batch(len(x_train))
W = tf.Variable(tf.random.normal((4, 1)), dtype=tf.float32)
b = tf.Variable(tf.random.normal((1,)), dtype=tf.float32)

# L2 Norm Function 정의
# https://junklee.tistory.com/29
def l2_loss(loss, beta = 0.01):
    # Weight Regularization * Beta (Weight 이상치의 영향을 최소화하는 정규화)
    W_reg = tf.nn.l2_loss(W) # output = sum(t ** 2) / 2
    loss = tf.reduce_mean(loss + W_reg * beta)
    return loss

# Flag가 True일때만 L2 Norm 으로 Cost 정의
def loss_fn2(hypothesis, features, labels, flag = False):
    cost = tf.reduce_mean(tf.square(hypothesis - labels))
    if(flag):
        cost = l2_loss(cost)
    return cost    

# Learning Rate Decay True
is_decay = True

# 0.1
starter_learning_rate = 0.1

# 50 Steap, 0.96, 이산 학습속도 감속 적용
# decayed_learning_rate = learning_rate * decay_rate ^ (global_step / decay_steps)
if(is_decay):    
    learning_rate = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=starter_learning_rate,
                                                                  decay_steps=50,
                                                                  decay_rate=0.96,
                                                                  staircase=True)
    optimizer = tf.keras.optimizers.SGD(learning_rate)
else:
    optimizer = tf.keras.optimizers.SGD(learning_rate=starter_learning_rate)

# Gradient Tape 정의
def grad(hypothesis, features, labels, l2_flag):
    with tf.GradientTape() as tape:
        loss_value = loss_fn2(linearReg_fn(features),features,labels, l2_flag)
    return tape.gradient(loss_value, [W,b]), loss_value

# 100회
EPOCHS = 101

for step in range(EPOCHS):
    for features, labels  in dataset:
        # X, Y Type Case
        features = tf.cast(features, tf.float32)
        labels = tf.cast(labels, tf.float32)
        # Gradient Tape
        grads, loss_value = grad(linearReg_fn(features), features, labels, False)
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))        
    # Verbose 10
    if step % 10 == 0:
        print("Iter: {}, Loss: {:.4f}".format(step, loss_value))

# 100회
EPOCHS = 101

for step in range(EPOCHS):
    for features, labels  in dataset:
        # X, Y Type Case
        features = tf.cast(features, tf.float32)
        labels = tf.cast(labels, tf.float32)
        # Gradient Tape
        grads, loss_value = grad(linearReg_fn(features), features, labels, True)
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))          
    # Verbose 10
    if step % 10 == 0:
        print("Iter: {}, Loss: {:.4f}".format(step, loss_value))

import numpy as np
import tensorflow as tf

tf.random.set_seed(777)  # for reproducibility
print(tf.__version__)

# MNIST Dataset 정의
mnist = tf.keras.datasets.mnist

# Train Data와 Test Data Load
(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

# Kera Model Data
model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(),
  tf.keras.layers.Dense(512, activation=tf.nn.relu),
  tf.keras.layers.Dropout(0.2),
  tf.keras.layers.Dense(10, activation=tf.nn.softmax)
])

# Model : Adam Optimizer / Cros Entropy Loss 선언
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# 5 Epoch 학습 수행
model.fit(x_train, y_train, epochs=5)

import matplotlib.pyplot as plt
%matplotlib inline

sample = 1
image = x_test[sample]

fig = plt.figure
plt.imshow(image, cmap='gray_r')
plt.show

num = 10
images = x_train[:num]
labels = y_train[:num]

num_row = 2
num_col = 5
fig, axes = plt.subplots(num_row,num_col, figsize = (1.5*num_col,2*num_row))

for i in range(num) :
  ax = axes[i//num_col, i%num_col]
  ax.imshow(images[i], cmap='gray_r')
  ax.set_title('Label: {}'.format(labels[i]))

plt.tight_layout()
plt.show()

# 모델 평가
model.evaluate(x_test, y_test)

img_index = 1
plt.imshow(x_test[img_index].reshape(28,28), cmap='gray_r')
pred = model.predict(x_test[img_index].reshape(1,28,28,1))
print(pred.argmax())

num = 100
images = x_test[:num]
labels = y_test[:num]

num_row = 10
num_col = 10
fig, axes = plt.subplots(num_row,num_col, figsize = (1.5*num_col,2*num_row))

for i in range(num) :
  ax = axes[i//num_col, i%num_col]
  ax.imshow(images[i], cmap='gray_r')
  pred = model.predict(x_test[i].reshape(1,28,28,1))  
  ax.set_title('Label: {}'.format(pred.argmax()))

plt.tight_layout()
plt.show()

print(x_test.shape)

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras

tf.random.set_seed(777)  # for reproducibility
print(tf.__version__)

# fashion_mnist 선언
fashion_mnist = tf.keras.datasets.fashion_mnist

# train_data, train_label, test_data, test_label 선언
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

# 0 - T-shirt/Top
# 1 - Trouser
# 2 - Pullover
# 3 - Dress
# 4 - Coat
# 5 - Sandal
# 6 - Shirt
# 7 - Sneaker
# 8 - Bag
# 9 - Ankle Boot
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

plt.figure()
plt.imshow(train_images[0])
plt.colorbar()
plt.grid(False)

num = 20
images = train_images[:num]/255.0
labels = train_labels[:num]/255.0

num_col = 10
num_row = int(num / num_col)

fig, axes = plt.subplots(num_row, num_col, figsize = (1.5*num_col, 2*num_row))

for i in range(num) :
  ax = axes[i//num_col, i%num_col]
  ax.imshow(train_images[i], cmap=plt.cm.binary)
  ax.spines['bottom'].set_color('white')
  ax.spines['top'].set_color('white')
  ax.yaxis.label.set_color('white')
  ax.xaxis.label.set_color('white')
  ax.title.set_color('white')
  ax.tick_params(axis='y', colors="white")  
  ax.tick_params(axis='x', colors='white')
  ax.set_title('Label: {}'.format(train_labels[i]))

plt.tight_layout()
plt.figure()
plt.show()

train_images = train_images / 255.0
test_images = test_images / 255.0

plt.figure(figsize=(10,10))
for i in range(25):
    plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(True)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])

model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)),
    keras.layers.Dense(128, activation=tf.nn.relu),
    keras.layers.Dense(10, activation=tf.nn.softmax)
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
model.fit(train_images, train_labels, epochs=5)

test_loss, test_acc = model.evaluate(test_images, test_labels)

print('Test accuracy:', test_acc)

import numpy as np
import tensorflow as tf
from tensorflow import keras

tf.random.set_seed(777)  # for reproducibility
print(tf.__version__)

# 50,000 Movie Reviews IMDB (10000개의 빈도수가 높은 단어를 학습시 Vector에 사용)
imdb = keras.datasets.imdb

# train_data, train_labels, test_data, test_labels
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)

# train_data, train_labels
print("Training entries: {}, labels: {}".format(len(train_data), len(train_labels)))
print(train_data[0])

# IMDB Data를 Vector을 실재 값으로 변환하여 출력
# A dictionary mapping words to an integer index
# 딕셔너리 단어를 인티져 인덱스로 맵핑
word_index = imdb.get_word_index()

# The first indices are reserved
# Data Check : Start 1 and 'the'
for k,v in word_index.items() :
  #print(len(k))
  if k == 'bad' : 
    print("K : ",k, " V : ", v+3)
  if v == 0 :
    print("K : ",k, " V : ", v)
  if v == 1 :
    print("K : ",k, " V : ", v)

# 최초 익덱스들은 Reserved : 0~3 예약어로 저장
word_index = {k:(v+3) for k,v in word_index.items()}
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # unknown
word_index["<UNUSED>"] = 3

reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])

# Word List : 0~3, 4~88587
print('check : ', reverse_word_index.get(0,'?'))
print('check : ', reverse_word_index.get(88587,'?'))
print(len(reverse_word_index))

# Review를 Decode
def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])

check_data_id = 3

print(len(train_data[check_data_id]))

# Train Data Value Print
print(train_data[check_data_id])

# Print label 
print(train_labels[check_data_id])

# Train Data Decode Print
decode_review(train_data[check_data_id])

# 학습과 평가를 위해 동일길이인 256길이의 단어로 PAD값을 주어 맞춰줌 (뒤의 길이는 0값으로 맞춰줌)
train_data = keras.preprocessing.sequence.pad_sequences(train_data,
                                                        value=word_index["<PAD>"],
                                                        padding='post',
                                                        maxlen=256)

test_data = keras.preprocessing.sequence.pad_sequences(test_data,
                                                       value=word_index["<PAD>"],
                                                       padding='post',
                                                       maxlen=256)

print(len(train_data[check_data_id]), len(test_data[check_data_id]))
print(train_data[check_data_id])

# Tensorflow keras API를 통해 모델에 대한 정의
# 입력 Size와 학습시킬 Layer의 크기와 Activation Function 정의
# input shape is the vocabulary count used for the movie reviews (10,000 words)
vocab_size = 10000

model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(16, activation=tf.nn.relu))
model.add(keras.layers.Dense(1, activation=tf.nn.sigmoid))

# Model Summary 출력
model.summary()

# Adam Optimizer과 Cross Entropy Loss 선언
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])

# 모델을 평가할 Test 데이타에 대한 정의(10000을 기준으로 학습과 평가 수행)
# 총 : 25000 평가 ~10000 / 학습 10001~
x_val = train_data[:10000]
partial_x_train = train_data[10000:]

y_val = train_labels[:10000]
partial_y_train = train_labels[10000:]

history = model.fit(partial_x_train,
                    partial_y_train,
                    epochs=40,
                    batch_size=512,
                    validation_data=(x_val, y_val),
                    verbose=1)

results = model.evaluate(test_data, test_labels)
print(results)

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import tensorflow as tf

tf.random.set_seed(777)  # for reproducibility

print(tf.__version__)

# XOR : 00 -> 0, 01 -> 1, 10 -> 1, 11 -> 0
x_data = [[0, 0],[0, 1],[1, 0],[1, 1]]
y_data = [[0]   ,   [1],   [1],   [0]]

plt.scatter(x_data[0][0],x_data[0][1], c='red' , marker='^')
plt.scatter(x_data[3][0],x_data[3][1], c='orange' , marker='^')
plt.scatter(x_data[1][0],x_data[1][1], c='blue' , marker='^')
plt.scatter(x_data[2][0],x_data[2][1], c='green' , marker='^')

plt.xlabel("x1")
plt.ylabel("x2")
plt.show()

# Tensorflow data API를 통해 학습시킬 값들을 담는다 (Batch Size는 한번에 학습시킬 Size로 정한다)
# preprocess function으로 features,labels는 실재 학습에 쓰일 Data 연산을 위해 Type를 맞춰준다
dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data)).batch(len(x_data))

def preprocess_data(features, labels):
    features = tf.cast(features, tf.float32)
    labels = tf.cast(labels, tf.float32)
    return features, labels

# Weight Bias 초기화
W = tf.Variable(tf.zeros((2,1)), name='weight')
b = tf.Variable(tf.zeros((1,)), name='bias')
print("W = {}, B = {}".format(W.numpy(), b.numpy()))

# Sigmoid를 가설로 선언
# 0과 1의 값만을 리턴 : tf.sigmoid(tf.matmul(X, W) + b)
def logistic_regression(features):
    hypothesis  = tf.divide(1., 1. + tf.exp(tf.matmul(features, W) + b))
    return hypothesis

# Cost 함수 정의
def loss_fn(hypothesis, features, labels):
    cost = -tf.reduce_mean(labels * tf.math.log(logistic_regression(features)) + (1 - labels) * tf.math.log(1 - hypothesis))
    return cost

optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)

# Sigmoid 함수를 통해 예측값이 0.5보다 크면 1을 반환하고 0.5보다 작으면 0으로 반환
def accuracy_fn(hypothesis, labels):
    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.float32))
    return accuracy

# GradientTape을 통해 경사값을 계산
def grad(hypothesis, features, labels):
    with tf.GradientTape() as tape:
        loss_value = loss_fn(logistic_regression(features),features,labels)
    return tape.gradient(loss_value, [W,b])

# epoch 1000회
EPOCHS = 1001

# epoch에서 1000회 step
for step in range(EPOCHS):
    # x, y
    for features, labels  in dataset:
        # features, labels를 받아 float을 맞춤
        features, labels = preprocess_data(features, labels)
        
        # gradient tape
        grads = grad(logistic_regression(features), features, labels)
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W,b]))

        # Verbose 100
        if step % 100 == 0:
            print("Iter: {}, Loss: {:.4f}".format(step, loss_fn(logistic_regression(features),features,labels)))

print("W = {}, B = {}".format(W.numpy(), b.numpy()))
x_data, y_data = preprocess_data(x_data, y_data)
test_acc = accuracy_fn(logistic_regression(x_data),y_data)
print("Testset Accuracy: {:.4f}".format(test_acc))

plt.scatter(x_data[0][0],x_data[0][1], c='red' , marker='^')
plt.scatter(x_data[3][0],x_data[3][1], c='orange' , marker='^')
plt.scatter(x_data[1][0],x_data[1][1], c='blue' , marker='^')
plt.scatter(x_data[2][0],x_data[2][1], c='green' , marker='^')
plt.plot(x_data,logistic_regression(x_data), c='gray')

plt.tick_params(axis='y', colors="white")  
plt.tick_params(axis='x', colors='white')

plt.xlabel("x1",color="white")
plt.ylabel("x2",color="white")
plt.show()

# W1,b1,W2,b2,W3,b3 정의
W1 = tf.Variable(tf.random.normal((2, 1)), name='weight1')
b1 = tf.Variable(tf.random.normal((1,)), name='bias1')
W2 = tf.Variable(tf.random.normal((2, 1)), name='weight2')
b2 = tf.Variable(tf.random.normal((1,)), name='bias2')
W3 = tf.Variable(tf.random.normal((2, 1)), name='weight3')
b3 = tf.Variable(tf.random.normal((1,)), name='bias3')

# layer1 : sigmoid ─┬─> layer3
# layer2 : sigmoid --┘
# hypothesis 가설
def neural_net(features):
    layer1 = tf.sigmoid(tf.matmul(features, W1) + b1)
    layer2 = tf.sigmoid(tf.matmul(features, W2) + b2)
    layer3 = tf.concat([layer1, layer2],-1)
    layer3 = tf.reshape(layer3, shape = [-1,2])
    hypothesis = tf.sigmoid(tf.matmul(layer3, W3) + b3)
    return hypothesis

# 손실함수
def loss_fn(hypothesis, labels):
    cost = -tf.reduce_mean(labels * tf.math.log(hypothesis) + (1 - labels) * tf.math.log(1 - hypothesis))
    return cost

# Optimizer SGD
optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)

# Accuracy 측정
def accuracy_fn(hypothesis, labels):
    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.float32))
    return accuracy

# Gradient Tape
def grad(hypothesis, features, labels):
    with tf.GradientTape() as tape:
        loss_value = loss_fn(neural_net(features),labels)
    return tape.gradient(loss_value, [W1, W2, W3, b1, b2, b3])

# epoch 50000
EPOCHS = 50000

for step in range(EPOCHS):
    for features, labels  in dataset:
        # preprocess 처리
        features, labels = preprocess_data(features, labels)
        
        # neural_net : layer1 + layer2 -> layer3
        grads = grad(neural_net(features), features, labels)

        # optimizer
        optimizer.apply_gradients(grads_and_vars=zip(grads,[W1, W2, W3, b1, b2, b3]))

        # verbose 5000
        if step % 5000 == 0:
            print("Iter: {}, Loss: {:.4f}".format(step, loss_fn(neural_net(features),labels)))

x_data, y_data = preprocess_data(x_data, y_data)

# test data
test_acc = accuracy_fn(neural_net(x_data),y_data)
print("Testset Accuracy: {:.4f}".format(test_acc))

plt.scatter(x_data[0][0],x_data[0][1], c='red' , marker='^')
plt.scatter(x_data[3][0],x_data[3][1], c='orange' , marker='^')
plt.scatter(x_data[1][0],x_data[1][1], c='blue' , marker='^')
plt.scatter(x_data[2][0],x_data[2][1], c='green' , marker='^')

plt.plot(x_data,neural_net(x_data), c='gray')

plt.tick_params(axis='y', colors="white")  
plt.tick_params(axis='x', colors='white')

plt.xlabel("x1",color="white")
plt.ylabel("x2",color="white")
plt.show()

# XOR 문제를 Deep Neural Network 활용 풀이
# dataset 
dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data)).batch(len(x_data))
nb_classes = 10

class wide_deep_nn():
  # 초기화
  def __init__(self, nb_classes):
      # 초기화
      super(wide_deep_nn, self).__init__()
      # W1,b1,W2,b2,W3,b3,W4,b4 정의
      self.W1 = tf.Variable(tf.random.normal((2, nb_classes)), name='weight1')
      self.b1 = tf.Variable(tf.random.normal((nb_classes,)), name='bias1')
      self.W2 = tf.Variable(tf.random.normal((nb_classes, nb_classes)), name='weight2')
      self.b2 = tf.Variable(tf.random.normal((nb_classes,)), name='bias2')
      self.W3 = tf.Variable(tf.random.normal((nb_classes, nb_classes)), name='weight3')
      self.b3 = tf.Variable(tf.random.normal((nb_classes,)), name='bias3')
      self.W4 = tf.Variable(tf.random.normal((nb_classes,1)), name='weight4')
      self.b4 = tf.Variable(tf.random.normal((1,)), name='bias4')
      self.variables = [self.W1, self.b1,self.W2, self.b2, self.W3, self.b3, self.W4, self.b4]

  # data preprocessing
  def preprocess_data(self, features, labels):
    features = tf.cast(features, tf.float32)
    labels = tf.cast(labels, tf.float32)
    return features, labels

  # 4Layer의 Neural Network를 통해 학습시킨 후 모델을 생성
  # layer1->layer2->layer3->hypothesis
  def deep_nn(self, features):
    layer1 = tf.sigmoid(tf.matmul(features, self.W1)+self.b1)
    layer2 = tf.sigmoid(tf.matmul(layer1, self.W2)+self.b2)
    layer3 = tf.sigmoid(tf.matmul(layer2, self.W3)+self.b3)
    hypothesis = tf.sigmoid(tf.matmul(layer3, self.W4)+self.b4)
    return hypothesis

  # loss function 
  def loss_fn(self, hypothesis, features, labels):
    cost = -tf.reduce_mean(labels*tf.math.log(hypothesis)+(1-labels)*tf.math.log(1-hypothesis))
    return cost

  # accuracy function
  def accuracy_fn(self, hypothesis,labels):
    predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
    accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.float32))
    return accuracy

  # Gradient Tape 
  def grad(self, hypothesis, features, labels):
    with tf.GradientTape() as tape:
      loss_value=self.loss_fn(self.deep_nn(features),features,labels)
    return tape.gradient(loss_value,self.variables)

  # epochs 2000, verbose 500
  def fit(self, dataset, EPOCHS=20000, verbose=500):
    optimizer = tf.keras.optimizers.SGD(learning_rate=0.01)
    for step in range(EPOCHS):
      for features, labels in dataset:
        features, labels = self.preprocess_data(features, labels)
        grads = self.grad(self.deep_nn(features),features,labels)
        optimizer.apply_gradients(grads_and_vars=zip(grads, self.variables))
        if step % verbose == 0:
          print("Iter: {}, Loss: {:.4f}".format(step, self.loss_fn(self.deep_nn(features),features,labels)))
  # Model Test
  def test_model(self,x_data,y_data):
    # Test Data preprocessing
    x_data,y_data = self.preprocess_data(x_data,y_data)
    # Accuracy check
    test_acc = self.accuracy_fn(self.deep_nn(x_data),y_data)
    # Accuracy Printing
    print("Testset Accuracy: {:.4f}".format(test_acc))

# model 정의
model = wide_deep_nn(nb_classes)

# fitting
model.fit(dataset)

# test data accuracy
model.test_model(x_data, y_data)

# dataset 정의
dataset = tf.data.Dataset.from_tensor_slices((x_data,y_data)).batch(len(x_data))

# data preprocessing 정의
def preprocess_data(features, labels):
  features = tf.cast(features, tf.float32)
  labels = tf.cast(labels, tf.float32)
  return features, labels

# summary 값을 logs폴더에 저장하고 아래 명령어로 실행해서 확인
# tensorboard --logdir=./logs/xor
log_path = "./logs/xor"
writer = tf.summary.create_file_writer(log_path)

# 위의 Data를 4Layer의 Neural Network를 통해 학습시킨 후 모델을 생성
# 각각의 값을 histogram으로 tensorboard에 저장 (Model)
# 각각의 값을 scalar값으로 tensorboard에 저장 (cost, accuracy)
# W1,b1,W2,b2,W3,b3,W4,b4 정의 및 선언
W1 = tf.Variable(tf.random.normal((2,10)), name='weight1')
b1 = tf.Variable(tf.random.normal((10,)), name='bias1')
W2 = tf.Variable(tf.random.normal((10,10)),name='weight2')
b2 = tf.Variable(tf.random.normal((10,)),name='bias2')
W3 = tf.Variable(tf.random.normal((10,10)),name='weight3')
b3 = tf.Variable(tf.random.normal((10,)),name='bias3')
W4 = tf.Variable(tf.random.normal((10,1)), name='weight4')
b4 = tf.Variable(tf.random.normal((1,)), name='bias4')

# layer1 -> layer2 -> layer3 -> hypothesis
def neural_net(features, step):
  layer1 = tf.sigmoid(tf.matmul(features, W1) + b1)
  layer2 = tf.sigmoid(tf.matmul(layer1, W2) + b2)
  layer3 = tf.sigmoid(tf.matmul(layer2, W3) + b3)
  hypothesis = tf.sigmoid(tf.matmul(layer3,W4) + b4)

  # tensorboard에 저장
  with writer.as_default():
    tf.summary.histogram("weights1",W1,step=step)
    tf.summary.histogram("biases1",b1,step=step)
    tf.summary.histogram("layer1",layer1,step=step)
    tf.summary.histogram("weights2",W2,step=step)
    tf.summary.histogram("biases2",b2,step=step)
    tf.summary.histogram("layer2",layer2,step=step)
    tf.summary.histogram("weight3",W3,step=step)
    tf.summary.histogram("biases3",b3,step=step)
    tf.summary.histogram("layer3",layer3,step=step)
    tf.summary.histogram("weights4",W4,step=step)
    tf.summary.histogram("biases4",b4, step=step)
    tf.summary.histogram("hypothesis",hypothesis,step=step)
  return hypothesis

# loss function
def loss_fn(hypothesis,labels):
  cost = -tf.reduce_mean(labels * tf.math.log(hypothesis) + (1-labels) * tf.math.log(1-hypothesis))
  with writer.as_default():
    tf.summary.scalar('loss',cost,step=step)
  return cost

# Optimizers SGD 정의 (Learning Rate 0.,1)
optimizer = tf.keras.optimizers.SGD(learning_rate=0.1)

# Accuracy Function
def accuracy_fn(hypothesis, labels):
  predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
  accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, labels), dtype=tf.float32))
  return accuracy

# GradientTape
def grad(hypothesis, features, labels, step):
  with tf.GradientTape() as tape:
    loss_value = loss_fn(neural_net(features, step),labels)
  return tape.gradient(loss_value, [W1,W2,W3,W4,b1,b2,b3,b4])

# Epochs 3000회
EPOCHS = 3000

# Step - 3000
for step in range(EPOCHS):
  # dataset x, y
  for features, labels in dataset:
    # data preprocessing
    features, labels = preprocess_data(features, labels)
    # gradient
    grads = grad(neural_net(features,step), features, labels, step)
    # optimizer
    optimizer.apply_gradients(grads_and_vars=zip(grads,[W1,W2,W3,W4,b1,b2,b3,b4]))

    # verbose 50
    if step % 50 == 0:
      loss_value = loss_fn(neural_net(features, step), labels)
      print("Iter: {}, Loss: {:4f}".format(step, loss_value))

# preprocessing
x_data, y_data = preprocess_data(x_data,y_data)

# accuracy
test_acc = accuracy_fn(neural_net(x_data,step),y_data)
print("Testset Accuracy: {:.4f}".format(test_acc))

# Jupyter Notebook에서 Tensorboard 실행
# Load the TensorBoard notebook extension
%load_ext tensorboard

'''Start TensorBoard through the command line or within a notebook experience.
The two interfaces are generally the same. In notebooks, use the %tensorboard line magic.
On the command line, run the same command without '%". '''

%tensorboard --logdir logs/xor

# tensorflow import
import tensorflow as tf
import numpy as np
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.datasets import mnist
from time import time
import os
print(tf.__version__)

# 전체 좋은 설명 : https://engineer-mole.tistory.com/m/18?category=911427
# Load : Check point function 
def load(model, checkpoint_dir):
    print(" [*] Reading checkpoints...")

    # check point state를 저장
    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)

    # ckpt가 존재하면 
    if ckpt :
        # ckpt_name 가져옴
        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)

        # checkpoint model
        checkpoint = tf.train.Checkpoint(dnn=model)

        # save path 지정 restore
        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))

        # check point split '-' countrer
        counter = int(ckpt_name.split('-')[1])

        print(" [*] Success to read {}".format(ckpt_name))
        return True, counter
    else:
        print(" [*] Failed to find a checkpoint")
        return False, 0

# folder check
def check_folder(dir):
    # 경로가 존재하지 않을 경우 생성
    if not os.path.exists(dir):
        os.makedirs(dir)
    return dir

# Data :Load
def load_mnist() :
    # train_x, train_y, test_x, test_y : mnist data load
    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()
    # train_data dimension 변경
    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]
    # train_data dimension 변경
    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]

    # train_data, test_data normalize
    # https://goodtogreate.tistory.com/entry/Neural-Network-%EC%A0%81%EC%9A%A9-%EC%A0%84%EC%97%90-Input-data%EB%A5%BC-Normalize-%ED%95%B4%EC%95%BC-%ED%95%98%EB%8A%94-%EC%9D%B4%EC%9C%A0
    train_data, test_data = normalize(train_data, test_data)

    # train_y, test_y 10 categorization
    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]
    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]

    return train_data, train_labels, test_data, test_labels

# Data pre-processing
def normalize(train_data, test_data):

    # 0~255를 0~1로 정규화
    train_data = train_data.astype(np.float32) / 255.0
    test_data = test_data.astype(np.float32) / 255.0

    return train_data, test_data

# loss function
def loss_fn(model, images, labels):
    # Logits 정의, Training = True 일경우 Dropout을 사용 (False는 반대)
    logits = model(images, training=True)    

    # loss = average ( cross_entropy( logits, labels ))
    # Category 분류의 문제 : categorical crossentropy
    # Output이 logit 상태가 아님으로 from_logits= True로 설정 
    # https://hwiyong.tistory.com/335 (softmax를 거치느냐 안거치느냐로 설명중)
    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, from_logits=True))
    return loss

# Accuracy Function
def accuracy_fn(model, images, labels):
    # Logits 정의
    logits = model(images, training=False)

    # Logits와 Label의 Equal 여부를 Prediction으로 정의
    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))

    # Prediction Boolean을 float으로 전환후 accuracy return
    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))
    return accuracy

# Gradient Tape
def grad(model, images, labels):
    with tf.GradientTape() as tape:
        # Loss Function 정의
        loss = loss_fn(model, images, labels)
    return tape.gradient(loss, model.variables)

# Model Function
# Layer Flatten 처리
def flatten() :
    return tf.keras.layers.Flatten()

# Layer의 Dense 설정
# units는 Output 채널 개수를 설정
# use_bias는 bias 사용 여부 설정
# kernel initializer는 weight 초기화
def dense(label_dim, weight_init) :
    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)

# Layer Activation Sigmoid
def sigmoid() :
    return tf.keras.layers.Activation(tf.keras.activations.sigmoid)

# Layer Activation Relu
def relu() :
    return tf.keras.layers.Activation(tf.keras.activations.relu)

# overfitting 방지를 위해 0~1사이의 rate로 dropout 사용을 설정
def dropout(rate) : 
    return tf.keras.layers.Dropout(rate)

# overfitting 방지를 위해 Batch Normalization을 사용하는 경우 정의
def batch_norm() : 
    return tf.keras.layers.BatchNormalization()

# model class 생성
# keras Model 상속 필요
class create_model_class(tf.keras.Model):
    # logits의 최종 아웃풋 개수 label_dim = 10
    def __init__(self, label_dim):
        # class 모델  초기화
        super(create_model_class, self).__init__()

        # Weight 초기화 : RandomNormal() 은 평균이 0 분산이 1인 가우시안 분포로 랜덤한 수를 생성하여 Weght로 설정
        # 10-1-1, 10-1-2
        # weight_init = tf.keras.initializers.RandomNormal()

        # 10-2-1 xavier
        # https://flonelin.wordpress.com/2018/01/28/weight-initalizer-%EC%A2%85%EB%A5%98/
        # Relu : 0이하의 신호를 제거
        # Glorot : input / output neuron 수에 기반 초기화의 스케일을 결정
        # He : Relu가 0 이하의 신호를 제거함으로 분산을 두배 주어 분산을 유지
        # Orthogonal : SVD(Singular Value Decomposition)으로 가중치 행렬 각행이 모두 수직으로 만듬
        weight_init = tf.keras.initializers.glorot_uniform()

        # 리스트 자료구조 타입
        self.model = tf.keras.Sequential()

        # Convolution을 이용할 경우 Flatten과정이 필요치 않음
        self.model.add(flatten())

        # [N,784] -> [N,256] -> [N,256]
        # 10-1-1, 10-1-2, 10-2-1
        # for i in range(2):
        # 10-2-2 Xavier Deep
        for i in range(4) :
            # 채널을 256으로 하고 Sigmoid를 쓸수있게 바꿈
            # 10-1-1, 10-1-2, 10-2-1
            # self.model.add(dense(256, weight_init))            
            # 10-2-2
            self.model.add(dense(512, weight_init))

            # 10-1-1 sigmoid 사용시
            # self.model.add(sigmoid())

            # 10-1-2 relu 사용시
            # self.model.add(relu())

            # 10-3 dropout 설정
            # self.model.add(dropout(rate=0.5))

            # 10-4 batch normalization 사용
            # drop out 사용시와 순서가 다름
            self.model.add(batch_norm())
            self.model.add(relu())
            
        # logit을 구할때 10개 output이 출력됨
        self.model.add(dense(label_dim, weight_init))

    # call 함수를 통해 아웃풀 출력을 정의
    def call(self, x, training=None, mask=None):
        x = self.model(x)
        return x

# Function 형태 Create Model
def create_model_function(label_dim) :
    # Weight 초기화
    # 10-1-1, 10-1-2
    # weight_init = tf.keras.initializers.RandomNormal()

    # 10-2-1 xavier
    weight_init = tf.keras.initializers.glorot_uniform()

    model = tf.keras.Sequential()
    model.add(flatten())

    # 10-1-1, 10-1-2, 10-2-1
    # for i in range(2) :
    # 10-2-2 Xavier Deep
    for i in range(4) :
        # 10-1-1, 10-1-2, 10-2-1
        # model.add(dense(256, weight_init))
        # 10-2-2
        model.add(dense(512,weight_init))

        # 10-1-1 sigmoid 사용시
        # model.add(sigmoid())
        
        # 10-1-2 relu 사용시
        # model.add(relu())
        
        # 10-3
        # model.add(dropout(rate=0.5))

        # 10-4 Batch Normalization
        model.add(batch_norm())
        model.add(relu())

    model.add(dense(label_dim, weight_init))

    return model

# Define data & hyper-parameter
# dataset load
""" dataset """
train_x, train_y, test_x, test_y = load_mnist()

# parameters 설정
""" parameters """
learning_rate = 0.001
batch_size = 128
training_epochs = 1

# 60000 / 128 = 468
training_iterations = len(train_x) // batch_size

label_dim = 10

train_flag = True

# data를 batch_size로 학습
# Shuffle : buffer_size는 input data 보다 커야함
# prefetch는 네트워크가 설정한 batch_size만큼 학습시 메모리에 batch_size만큼 적재
# batch_size 설정
# .\ 이후 space에 에러 출력됨
""" Graph Input using Dataset API """
train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\
    shuffle(buffer_size=100000).\
    prefetch(buffer_size=batch_size).\
    batch(batch_size, drop_remainder=True)

test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\
    shuffle(buffer_size=100000).\
    prefetch(buffer_size=len(test_x)).\
    batch(len(test_x))

# 모델 / 옵티마이저 / 롸이터 정의
""" Model """
network = create_model_function(label_dim)

# Adam Optimizer
""" Training """
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

# Writer 정의
# 10-1-1 sigmoid 사용시
# model_dir = 'nn_softmax'
# 10-1-2 relu 사용시
# model_dir = 'nn_relu'
# 10-2-1 xavier 사용시
# model_dir = 'nn_xavier'
# 10-2-2 xavier deep
# model_dir = 'nn_deep'
# 10-3 relu dropout
# model_dir = 'nn_dropout'
# 10-4 Batch Normalization

""" Writer """
checkpoint_dir = 'checkpoints'
logs_dir = 'logs'

model_dir = 'nn_batchnorm'

checkpoint_dir = os.path.join(checkpoint_dir, model_dir)
check_folder(checkpoint_dir)
checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)
logs_dir = os.path.join(logs_dir, model_dir)

# Train / Test 여부에 따라 분기
if train_flag :
    # 체크포인트는 재학습시 변경되었던 Weight를 복원함
    checkpoint = tf.train.Checkpoint(dnn=network)

    # 텐저보드를 위한 롸이터 생성
    # create writer for tensorboard
    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)

    # 시작 시간
    start_time = time()

    # 체크포인트 존재시 Resotre
    # restore check-point if it exits
    could_load, checkpoint_counter = load(network, checkpoint_dir)    

    # Check point 존재시 해당 값으로 설정
    if could_load:
        # start_epoch = (int)(checkpoint_counter / training_iterations)        
        # counter = checkpoint_counter    
        start_epoch = 0
        start_iteration = 0
        counter = 0
        print(" [*] Load SUCCESS")
    else:
        start_epoch = 0
        start_iteration = 0
        counter = 0
        print(" [!] Load failed...")
    
    # train phase
    with summary_writer.as_default():  # for tensorboard
        # epoch와 iteration에 대한 중첩 for 문 수행
        for epoch in range(start_epoch, training_epochs):
            for idx, (train_input, train_label) in enumerate(train_dataset):            
                grads = grad(network, train_input, train_label)
                # optimizer.apply_gradients를 호출하여 gradient를 적용하여 네트워크를 학습
                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))

                # Loss Function
                train_loss = loss_fn(network, train_input, train_label)

                # Accuracy Function
                train_accuracy = accuracy_fn(network, train_input, train_label)
                
                # Test Dataset
                for test_input, test_label in test_dataset:                
                    test_accuracy = accuracy_fn(network, test_input, test_label)

                # Summary Scalar형 텐서 사용
                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)
                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)
                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)

                print(
                    "Epoch: [%2d] [%5d/%5d] time: %4.2f, tr_loss: %.4f, tr_acc: %.4f, ts_acc: %.4f" \
                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,
                       test_accuracy))
                counter += 1             
        # 체크 포인트 저장
        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))
        
# test phase      
else :
    _, _ = load(network, checkpoint_dir)
    for test_input, test_label in test_dataset:    
        test_accuracy = accuracy_fn(network, test_input, test_label)

    print("test_Accuracy: %.4f" % (test_accuracy))

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
from tensorflow import keras
import matplotlib.pyplot as plt

print(tf.__version__)
print(keras.__version__)

# 1X3X3X1 shape
image = tf.constant([ [ [ [1],[2],[3] ],[ [4],[5],[6] ], [ [7],[8],[9] ] ] ], dtype=np.float32)
print(image.shape)

# Grey Shape
plt.imshow(image.numpy().reshape(3,3), cmap='Greys')
plt.show()

# 12,16,24,28 shape -> plt.show (check)
check = tf.constant([ [ [ [12.] ],[ [16.] ] ],[ [ [24.] ],[ [28.] ] ] ], dtype=np.float32)
plt.imshow(check.numpy().reshape(2,2), cmap='Greys')
plt.show()

# Image : 1,3,3,1  Filter : 2,2,1,1  Stride : 1X1  Padding : Valid
# 1  2  3    1  1    12  16
# 4  5  6  + 1  1 -> 24  28
# 7  8  9 
# 1 filter (22,11) with padding : valid
print("image.shape", image.shape)

# Weight 설정
weight = np.array([ [ [ [1.] ],[ [1.] ] ],[ [ [1.] ],[ [1.] ] ] ])
print("weight.shape", weight.shape)

weight_init = tf.constant_initializer(weight)
conv2d = keras.layers.Conv2D(filters=1, kernel_size=2, padding='VALID', kernel_initializer=weight_init)(image)

print("conv2d.shape", conv2d.shape)
print(conv2d.numpy().reshape(2,2))

plt.imshow(conv2d.numpy().reshape(2,2), cmap='gray')
plt.show()

# Image : 1,3,3,1  Filter : 2,2,1,1  Stride : 1X1  Padding : SAME (3X3)
# 1  2  3  0      1  1      12  16   9
# 4  5  6  0  ->  1  1  ->  24  28  15
# 7  8  9  0                15  17   9
# 0  0  0  0 
print("image.shape", image.shape)

weight = np.array([[[[1.]],[[1.]]],[[[1.]],[[1.]]]])
print("weight.shape", weight.shape)

weight_init = tf.constant_initializer(weight)
conv2d = keras.layers.Conv2D(filters=1, kernel_size=2, padding='SAME', kernel_initializer=weight_init)(image)

print("conv2d.shape", conv2d.shape)
print(conv2d.numpy().reshape(3,3))

plt.imshow(conv2d.numpy().reshape(3,3), cmap='gray')
plt.show()

# image shape 1,3,3,1
# print("imag:\n", image)
print("image.shape", image.shape)

# wieght shape 2,2,1,3
weight = np.array([ [ [ [1.,10.,-1.] ] ,
                      [ [1.,10.,-1.] ] ] , 
                    [ [ [1.,10.,-1.] ],
                      [ [1.,10.,-1.] ] ] ])
print("weight.shape", weight.shape)
weight_init = tf.constant_initializer(weight)

################################################################################
# Image : 1,3,3,1  Filter : 2,2,1,(1)  Stride : 1X1  Padding : SAME (3X3)
# 1  2  3        1   1      12  16   9
# 4  5  6   ->   1   1  ->  24  28  15
# 7  8  9                   15  17   9
################################################################################
# Image : 1,3,3,1  Filter : 2,2,1,(2)  Stride : 1X1  Padding : SAME (3X3)
# 1  2  3       10  10      120 160  90
# 4  5  6   ->  10  10  ->  240 280 150
# 7  8  9                   150 170  90
################################################################################
# Image : 1,3,3,1  Filter : 2,2,1,(3)  Stride : 1X1  Padding : SAME (3X3)
# 1  2  3       -1  -1      -12 -16  -9
# 4  5  6   ->  -1  -1  ->  -24 -28 -15
# 7  8  9                   -15 -17  -9
conv2d = keras.layers.Conv2D(filters=3, kernel_size=2, padding='SAME', kernel_initializer=weight_init)(image)
print("conv2d.shape", conv2d.shape)

# 행/열 교체 : 0 -> 4차원 / 3 -> 1차원 
feature_maps = np.swapaxes(conv2d, 0, 3)
print("feature_maps", feature_maps.shape)

# swapaxes / reshape / tranpose 
print("weight", weight.shape)
weight_swap = np.swapaxes(weight, 2, 3) # 2,2,1,3 -> 2,2,3,1
print("weight_swap1", weight_swap.shape)
weight_swap = np.swapaxes(weight, 0, 3) # 2,2,3,1 -> 1,2,3,2
print("weight_swap2", weight_swap.shape)
weight_reshape = weight.reshape(2,3,-1) # 2,2,1,3 -> 1,2,3,2
print("weight_reshape1", weight_reshape.shape)
weight_reshape = weight.reshape(1,-1) # 2,2,1,3 -> 1,2,3,2
print("weight_reshape2", weight_reshape.shape)
weight_trans = np.transpose(weight)
print("weight_trans", weight_trans.shape)

for i, feature_map in enumerate(feature_maps):
    print(feature_map.reshape(3,3))
    plt.subplot(1,3,i+1), plt.imshow(feature_map.reshape(3,3), cmap='gray')
plt.show()

image = tf.constant([ [ [ [1], [1], [2], [4] ],
                        [ [5], [6], [7], [8] ],
                        [ [3], [2], [1], [0] ],
                        [ [1], [2], [3], [4] ]  ] ], dtype=np.float32)
print(image.shape)

#  1  1  2  4      
#  5  6  7  8  ->   6   7   8
#  3  2  1  0       6   7   8
#  1  2  3  4       3   3   4
pool = keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='VALID')(image)
print(pool.shape)
print(pool.numpy())

#  1  1  2  4      
#  5  6  7  8  ->   6   8
#  3  2  1  0       3   4
#  1  2  3  4       
pool = keras.layers.MaxPool2D(pool_size=(2,2), strides=2, padding='VALID')(image)
print(pool.shape)
print(pool.numpy())

# image : 1X2X2X1
#  4  3  -> 4
#  2  1
image = tf.constant([ [ [ [4], [3] ],
                          [ [2],[1] ] ] ], dtype=np.float32)
print(image.shape)

pool = keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='VALID')(image)
print(pool.shape)
print(pool.numpy())

# image : 1X2X2X1
#  4  3  0  ->  4  3
#  2  1  0      2  1
#  0  0  0
image = tf.constant([[[[4],[3]],[[2],[1]]]], dtype=np.float32)
pool = keras.layers.MaxPool2D(pool_size=(2,2), strides=1, padding='SAME')(image)
print(pool.shape)
print(pool.numpy())

# header
mnist = keras.datasets.mnist
class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

#mnist = keras.datasets.fashion_mnist
#class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# load data
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()

# nomalizaiton
train_images = train_images.astype(np.float32) / 255.
test_images = test_images.astype(np.float32) / 255.

# print image 0
img = train_images[0]
plt.imshow(img, cmap='gray')
plt.show()

# img redefine
img = train_images[0]

# img reshape
print("Before reshape", img.shape)
img = img.reshape(-1,28,28,1)
print("After reshape", img.shape)

# image를 tensor로 정의
# 벡터 축에 대한 행렬을 일반화
img = tf.convert_to_tensor(img)

# 변형) Random 대신 지정
# 초기 Weight Random 저장
# weight_init = keras.initializers.RandomNormal(stddev=0.01)

# wieght shape 3,3,1,5
weight = np.array([ [ [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ] ],
                    [ [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ] ],
                    [ [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ], [ [ 1.,0.01,-1.,0.,10.] ] ] ])
print("weight.shape", weight.shape)
weight_init = tf.constant_initializer(weight)

# 1X28X28X1 -> 2X2 -> 1X14X14X5
conv2d = keras.layers.Conv2D(filters=5, kernel_size=3, strides=(2, 2), padding='SAME', kernel_initializer=weight_init)(img)
print(conv2d.shape)

# 4차원 - 1차원 Swap
print("Before swapaxes : ",conv2d.shape)
feature_maps = np.swapaxes(conv2d, 0, 3)
print("After swapaxes : ",feature_maps.shape)

# Filter 수만큼 For loop
for i, feature_map in enumerate(feature_maps):
    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(14,14), cmap='gray')
plt.show()

# maxpool 수행
pool = keras.layers.MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='SAME')(conv2d)
print(pool.shape)

# swap 축
feature_maps = np.swapaxes(pool, 0, 3)

# filter 만큼 for loop
for i, feature_map in enumerate(feature_maps):
    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(7, 7), cmap='gray')
plt.show()

# img redefine
img = train_images[0]

# img reshape
print("Before reshape", img.shape)
img = img.reshape(-1,28,28,1)
print("After reshape", img.shape)

# image를 tensor로 정의
# 벡터 축에 대한 행렬을 일반화
img = tf.convert_to_tensor(img)

# 초기 Weight Random 저장
weight_init = keras.initializers.RandomNormal(stddev=0.01)

# 1X28X28X1 -> 2X2 -> 1X14X14X5
conv2d = keras.layers.Conv2D(filters=5, kernel_size=3, strides=(2, 2), padding='SAME', kernel_initializer=weight_init)(img)
print(conv2d.shape)

# 4차원 - 1차원 Swap
print("Before swapaxes : ",conv2d.shape)
feature_maps = np.swapaxes(conv2d, 0, 3)
print("After swapaxes : ",feature_maps.shape)

# Filter 수만큼 For loop
for i, feature_map in enumerate(feature_maps):
    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(14,14), cmap='gray')
plt.show()

pool = keras.layers.MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding='SAME')(conv2d)
print(pool.shape)

feature_maps = np.swapaxes(pool, 0, 3)
for i, feature_map in enumerate(feature_maps):
    plt.subplot(1,5,i+1), plt.imshow(feature_map.reshape(7, 7), cmap='gray')
plt.show()

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
import numpy as np
import matplotlib.pyplot as plt
import os

print(tf.__version__)
print(keras.__version__)

# Hyper Parameters 설정
learning_rate = 0.001
training_epochs = 15
batch_size = 100

tf.random.set_seed(777)

# Creating a Checkpoint Directory 
cur_dir = os.getcwd()
print(cur_dir)

ckpt_dir_name = 'checkpoints'
model_dir_name = 'minst_cnn_seq'

checkpoint_dir = os.path.join(cur_dir, ckpt_dir_name, model_dir_name)
print(checkpoint_dir)

os.makedirs(checkpoint_dir, exist_ok=True)

checkpoint_prefix = os.path.join(checkpoint_dir, model_dir_name)
print(checkpoint_prefix)

## MNIST Dataset #########################################################
mnist = keras.datasets.mnist
class_names = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']
##########################################################################

## Fashion MNIST Dataset #################################################
#mnist = keras.datasets.fashion_mnist
#class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
##########################################################################

# Datasets Load
(train_images, train_labels), (test_images, test_labels) = mnist.load_data()    
    
# Train Data Normalization
train_images = train_images.astype(np.float32) / 255.
test_images = test_images.astype(np.float32) / 255.

# Train Data 마지막 차원 추가
print("Before : ",train_images.shape)
train_images = np.expand_dims(train_images, axis=-1)
test_images = np.expand_dims(test_images, axis=-1)
print("After : ",train_images.shape)

# Label 카테고리 지정
# Converts a class vector (integers) to binary class matrix.
train_labels = to_categorical(train_labels, 10)
test_labels  = to_categorical(test_labels, 10)    

# 이미지 경로 또는 RAW 데이터를 TF를 사용해 DataSet으로 전환
train_dataset = tf.data.Dataset.from_tensor_slices((train_images, train_labels)).shuffle(buffer_size=100000).batch(batch_size)
test_dataset  = tf.data.Dataset.from_tensor_slices((test_images , test_labels )).batch(batch_size)

# 모델 생성 (sequential-eager)
def create_model_seq():
    model = keras.Sequential()
    
    # 28 X 28 X 32
    model.add(keras.layers.Conv2D(filters=32, kernel_size=3, activation=tf.nn.relu, padding='SAME', input_shape=(28, 28, 1)))
    model.add(keras.layers.MaxPool2D(padding='SAME'))
    
    # 14 X 14 X 64
    model.add(keras.layers.Conv2D(filters=64, kernel_size=3, activation=tf.nn.relu, padding='SAME'))
    model.add(keras.layers.MaxPool2D(padding='SAME'))

    # 7 X 7 X 128
    model.add(keras.layers.Conv2D(filters=128, kernel_size=3, activation=tf.nn.relu, padding='SAME'))
    model.add(keras.layers.MaxPool2D(padding='SAME'))
    
    # 4 X X 128 -> 2048
    model.add(keras.layers.Flatten())

    # 256
    model.add(keras.layers.Dense(256, activation=tf.nn.relu))
    
    # Dropout 0.4
    model.add(keras.layers.Dropout(0.4))

    # 10
    model.add(keras.layers.Dense(10))
    return model

def create_model_fuc() :
    inputs = keras.Input(shape=(28,28,1))
    conv1 = keras.layers.Conv2D(filters=32, kernel_size=[3,3],padding='SAME', activation=tf.nn.relu)(inputs)
    pool1 = keras.layers.MaxPool2D(padding='SAME')(conv1)
    conv2 = keras.layers.Conv2D(filters=64, kernel_size=[3,3],padding='SAME', activation=tf.nn.relu)(pool1)
    pool2 = keras.layers.MaxPool2D(padding='SAME')(conv2)
    conv3 = keras.layers.Conv2D(filters=128, kernel_size=[3,3],padding='SAME', activation=tf.nn.relu)(pool2)
    pool3 = keras.layers.MaxPool2D(padding='SAME')(conv3)
    pool3_flat = keras.layers.Flatten()(pool3)
    dense4 = keras.layers.Dense(units=256, activation=tf.nn.relu)(pool3_flat)
    drop4 = keras.layers.Dropout(rate=0.4)(dense4)
    logits = keras.layers.Dense(units=10)(drop4)
    return keras.Model(inputs=inputs, outputs=logits)

#model = create_model_seq()
model = create_model_fuc()

# Model Summary 출력
model.summary()

# tf.function : decorator 
# 즉시 실행이 가능한 Tensorflow Graph로 전환
# Loss Function 
@tf.function
def loss_fn(model, images, labels):
    logits = model(images, training=True)
    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(
        y_pred=logits, y_true=labels, from_logits=True))    
    return loss   

# Calculating Gradient
@tf.function
def grad(model, images, labels):
    with tf.GradientTape() as tape:
        loss = loss_fn(model, images, labels)
    return tape.gradient(loss, model.variables)

# Calculating Model's Accuracy
@tf.function
def evaluate(model, images, labels):
    logits = model(images, training=False)
    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    return accuracy

# ADAM Optimizer
# Learning_rate = 0.001
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)

# Creating Checkpoint
checkpoint = tf.train.Checkpoint(cnn=model)

# Train Function Define
@tf.function
def train(model, images, labels):
    grads = grad(model, images, labels)
    optimizer.apply_gradients(zip(grads, model.trainable_variables))

# Train Model
print('Learning Started...')

# Epoch만큼 for loop
for epoch in range(training_epochs):
    # default setting
    avg_loss = 0.
    avg_train_acc = 0.
    avg_test_acc = 0.
    train_step = 0
    test_step = 0    
    
    # train_dataset에서 for loop
    for images, labels in train_dataset:
        # train
        train(model, images, labels)
        #grads = grad(model, images, labels)                
        #optimizer.apply_gradients(zip(grads, model.variables))
        
        # loss
        loss = loss_fn(model, images, labels)
        
        # evaluate
        acc = evaluate(model, images, labels)
        
        # loss cum
        avg_loss = avg_loss + loss
        
        # acc cum
        avg_train_acc = avg_train_acc + acc
        
        # for loop
        train_step += 1
    
    # avg_loss  / avg_train_acc
    avg_loss = avg_loss / train_step
    avg_train_acc = avg_train_acc / train_step
    
    # test_dataset에서 for loop
    for images, labels in test_dataset:        
        # evaluate 
        acc = evaluate(model, images, labels)        
        avg_test_acc = avg_test_acc + acc
        test_step += 1    
    
    # test acc average
    avg_test_acc = avg_test_acc / test_step    

    print('EPCH : ', '{}'.format(epoch + 1), '| LOSS : ', '{:.8f}'.format(avg_loss), '| TR_ACC : ', '{:.4f}'.format(avg_train_acc), '| TS_ACC : ', '{:.4f}'.format(avg_test_acc))
    
    checkpoint.save(file_prefix=checkpoint_prefix)

print('Learning Finished...')