import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

V, returns = np.zeros(5), [[] for _ in range(5)]

for _ in range(300):
    s, episode = 0, []
    while s < 4:
        sn, r = s + 1, float(s == 3)
        episode.append((s, r))
        s = sn

    G = 0
    for state, reward in reversed(episode):
        G = reward + 0.9 * G
        if not any(x[0] == state for x in episode[:episode.index((state, reward))]):
            returns[state].append(G)
            V[state] = np.mean(returns[state])

pd.Series(V).plot(kind="bar", title="First-Visit Monte Carlo State Values")
plt.xlabel("State")
plt.ylabel("Value")
plt.show()