rng = jr.PRNGKey(779)
ngenerators = 10
rnglist = jr.split(rng, ngenerators)WARNING:2025-07-10 10:59:22,813:jax._src.xla_bridge:794: An NVIDIA GPU may be present on this machine, but a CUDA-enabled jaxlib is not installed. Falling back to cpu.
In this notebook, we demonstrate how we utilize random features to disentangle the size of the Dense Associative Memory network from the number of memories to be stored. Given the standard log-sum-exp energy \(E_\beta(\cdot; \boldsymbol{\Xi})\), corresponding to a model \(f_\boldsymbol{\Xi}\) of size \(O(DK)\), we demonstrate how we can use the trigonometric random features to develop an approximate energy \(\tilde{E}_\beta(\cdot; \mathbf{T})\) using a distributed representation \(\mathbf{T}\) of the memories \(\boldsymbol{\Xi} = \{ \boldsymbol{\xi}^\mu, \mu \in [\![ K ]\!] \}\), thus giving us a model \(f_{\mathbf{T}}\) of size \(O(Y)\).
For further details on this work, please see the following publication: > Hoover, B., Chau, D.H., Strobelt, H., Ram, P. and Krotov, D., 2023. Dense Associative Memory Through the Lens of Random Features. Advances in Neural Information Processing Systems. [paper] [GitHub]
WARNING:2025-07-10 10:59:22,813:jax._src.xla_bridge:794: An NVIDIA GPU may be present on this machine, but a CUDA-enabled jaxlib is not installed. Falling back to cpu.
def plot_energy_landscape(
energies,
plt_ax,
bounds,
colormap=CMAP,
heatmap_alpha=HMA,
):
assert energies.shape[0] == energies.shape[1]
plt_ax.imshow(
energies, extent=bounds,
origin='lower', cmap=colormap,
alpha=heatmap_alpha,
)
plt_ax.tick_params(
axis='both', which='both',
top=False, bottom=False,
labelbottom=False, labelleft=False,
left=False, right=False
)Consider a set of memories \(\boldsymbol{\Xi} = \{ \boldsymbol{\xi}^1, \ldots, \boldsymbol{\xi}^K \}\) where each memory \(\boldsymbol{\xi}^\mu \in \mathbb{R}^D\) is a vector in a \(D\)-dimensional Euclidean space. For a state vector \(\mathbf{v} \in \mathbb{R}^D\), the commonly used log-sum-exp energy is given by \[ E_\beta( \mathbf{v}; \boldsymbol{\Xi} ) = - \frac{1}{\beta} \log \sum_{\mu = 1}^K \exp \left(- \frac{\beta}{2} \left\Vert \mathbf{v} - \boldsymbol{\xi}^\mu \right \Vert^2 \right), \] where \(\beta > 0\) is the inverse-temperature controlling the sharpness of the energy near the memories, with larger values of \(\beta\) implying sharper energy landscapes, while smaller values induce smoother ones.
We can implement this energy function as follows
betas = [0.01, 0.1, 1, 10]
figscaler = 2
fig, axs = plt.subplots(
1, len(betas), figsize=(
len(betas) * figscaler,
figscaler
),
sharex=True, sharey=True
)
beta_en_map = {}
for b, ax in tqdm(
zip(betas, axs), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
en = np.zeros_like(V[0])
for i, j in product(range(nsteps), range(nsteps)):
en[i,j] = lse_energy(V[:, i, j], Xi, b)
beta_en_map[b] = en
plot_energy_landscape(
en, ax, np.array([xmin, xmax, ymin, ymax])
)
plot_states(Xi, ax, marker='*', color=MCOLOR)
ax.set_title(r"$\beta$" + f":{b:0.2f}")
plt.show()100%|███████████████| 4/4 [00:04<00:00, 1.08s/it]

For an initial state vector \(\mathbf{q} \in \mathbb{R}^D\), we can minimize its energy utilizing the energy gradient. Initializing the energy descent at the \(\mathbf{q}\), that is \(\mathbf{v}^{(0)} \gets \mathbf{q}\), we perform the following gradient descent steps for \(T\) iterations: \[ \mathbf{v}^{(t)} \gets \mathbf{v}^{(t-1)} - \alpha \nabla_{\mathbf{v}} E_\beta( \mathbf{v}^{(t-1)}; \boldsymbol{\Xi} ), \] for \(t = 1, \ldots, T\) with \(\alpha > 0\) as the step-size (or learning rate) for the energy descent. The final \(\mathbf{v}^{(T)}\) is the output of the model.
We can implement this using auto-differentiation in JAX by computing the gradient of the lse_energy function with respect to its input state.
def lse_energy_descent(
q: Float[Array, "D"],
memories: Float[Array, "K D"],
beta,
energy_fn,
depth: int=10,
alpha: float = 0.01,
return_grads=False,
clamp_idxs: Optional[Bool[Array, "D"]]=None
) -> Float[Array, "D"]:
"""Energy descent with the LSE energy"""
# en = lambda state: energy_fn(state, memories, beta)
dEdxf = jax.jit(jax.value_and_grad(energy_fn))
logs = {}
def step(x, i):
E, dEdx = dEdxf(x, memories, beta)
if clamp_idxs is not None:
dEdx = jnp.where(clamp_idxs, 0, dEdx)
x = x - alpha * dEdx
aux = (E, dEdx) if return_grads else (E,)
return x, aux
x, aux = jax.lax.scan(step, q, jnp.arange(depth))
logs['energies'] = aux[0]
if return_grads:
logs['grads'] = aux[1]
return x, logsAs an example, we will use the previous set of memories to perform the energy descent for a randomly generated query \(\mathbf{q} \in \mathbb{R}^D\) using the above function lse_energy_descent. We will note the intermediate states \(\mathbf{v}^{(0)}, \ldots, \mathbf{v}^{(T)}\), and the energy \(E_\beta(\mathbf{v}^{(t)}; \boldsymbol{\Xi}), t \in [\![ T ]\!]\) at each layer of the DenseAM (equivalently, the energy at each iteration of the energy gradient descent).
In the following example, the number of DenseAM layers (equivalently, the number of energy descent steps) is set at \(T = 1000\), and we use a step-size \(\alpha = 0.01\). We will plot the intermediate states at every NUPDATES=25 layers.
NSTATES = 20
NUPDATES = 25
NQUERIES = 3
ALPHA = 0.01
rngidx = 8
fig, axs = plt.subplots(
NQUERIES+1, len(betas), figsize=(
len(betas) * figscaler,
NQUERIES+2 * figscaler
),
sharex="row",
)
beta_true_states_en_map = {}
for bidx, b in tqdm(
enumerate(betas), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
plot_energy_landscape(
beta_en_map[b], axs[0, bidx],
np.array([xmin, xmax, ymin, ymax])
)
plot_states(
Xi, axs[0, bidx],
marker='*', color=MCOLOR
)
axs[0, bidx].set_title(r"$\beta$" + f":{b:0.2f}")
# Randomly generating a query
queries = jr.uniform(
rnglist[rngidx], (NQUERIES, D)
) * 2 * maxabs - maxabs
beta_cache = []
for qidx, query in enumerate(queries):
qstates = [query]
qens = []
# Perform energy descent
for i in range(NSTATES):
query, logs = lse_energy_descent(
query, Xi, b, lse_energy,
depth=NUPDATES, alpha=ALPHA
)
qstates += [query]
qens += [logs['energies']]
qstates = np.array(qstates)
plot_states(
qstates, axs[0, bidx],
marker='o', color=QCOLOR
)
qens = np.array(qens).reshape(-1)
plot_energy_descent(
qens, axs[qidx+1, bidx], color=QCOLOR
)
beta_cache += [(qstates, qens)]
beta_true_states_en_map[b] = beta_cache #(qstates, qens)
fig.tight_layout()
plt.show()100%|███████████████| 4/4 [00:10<00:00, 2.69s/it]

It is easy to see that the aforemention energy function can be viewed as a kernel sum. Specifying a kernel function \(\kappa: \mathbb{R}^D \times \mathbb{R}^D \to \mathbb{R}\) such that \(\kappa(\mathbf{x}, \mathbf{x}') = \exp(-\frac{1}{2} \Vert \mathbf{x} - \mathbf{x}' \Vert^2 )\), the radial basis function or RBF kernel, we can reduce the energy function to a kernel sum as follows: \[\begin{align} E_\beta( \mathbf{v}; \boldsymbol{\Xi} ) & = - \frac{1}{\beta} \log \sum_{\mu = 1}^K \underbrace{ \exp \left(- \frac{\beta}{2} \left\Vert \mathbf{v} - \boldsymbol{\xi}^\mu \right \Vert^2 \right) }_{\kappa(\sqrt{\beta}\mathbf{v}, \sqrt{\beta}\boldsymbol{\xi}^\mu)} \\ & = - \frac{1}{\beta} \log \sum_{\mu = 1}^K \kappa \left(\sqrt{\beta}\mathbf{v}, \sqrt{\beta}\boldsymbol{\xi}^\mu \right), \end{align}\]
In general, we need to keep around all the memories \(\boldsymbol{\Xi} = \left\{ \boldsymbol{\xi}^\mu \in \mathbb{R}^D, \mu \in [\![ K ]\!] \right\}\) to compute this kernel sum, and thus the energy function.
However, if there exists a feature map \(\Phi: \mathbb{R}^D \to \mathbb{R}^Y\), such that, the dot-product in this feature space approximates the kernel function as follows: \[ \kappa(\mathbf{x}, \mathbf{x}') \approx \left\langle \Phi(\mathbf{x}), \Phi(\mathbf{x}') \right\rangle, \] then the kernel sum can be simplified as (dropping the \(\beta\) for now) \[\begin{align} \sum_{\mu = 1}^K \kappa(\mathbf{v}, \boldsymbol{\xi}^\mu) & \approx \sum_{\mu = 1}^K \left\langle \Phi(\mathbf{v}), \Phi(\boldsymbol{\xi}^\mu) \right\rangle \\ & = \left\langle \Phi(\mathbf{v}), \sum_{\mu = 1}^K \Phi(\boldsymbol{\xi}^\mu) \right\rangle \\ & = \left\langle \Phi(\mathbf{v}), \mathbf{T} \right\rangle, \qquad \text{where} \quad \mathbf{T} = \sum_{\mu = 1}^K \Phi(\boldsymbol{\xi}^\mu). \end{align}\] In this case, we need to compute the vector \(\mathbf{T} \in \mathbb{R}^Y\) using all the \(K\) memories \(\left\{ \boldsymbol{\xi}^\mu, \mu \in [\![ K ]\!] \right\}\) just once, and then use \(\mathbf{T}\) for all subsequent kernel sum approximation without needing access to the original memories.
Various approximate feature maps have been developed for the RBF kernel. The first feature map proposed by Rahimi & Recht (2007) utilizes random features and trigonometric function. More recently, Choromanski et al. (2021) have proposed positive random features utilizing the exponential function. Both these random features are presented below, where \(\mathcal{N}(0, \mathbf{I}_D)\) is the \(D\)-dimensional multivariate isotropic standard normal distribution:
\[ \Phi(\mathbf{x}) = \frac{1}{\sqrt{Y}} \left[ \begin{array}{c} \cos \langle \boldsymbol{\omega}^1, \mathbf{x} \rangle \\ \sin \langle \boldsymbol{\omega}^1, \mathbf{x} \rangle \\ \cos \langle \boldsymbol{\omega}^2, \mathbf{x} \rangle \\ \sin \langle \boldsymbol{\omega}^2, \mathbf{x} \rangle \\ \cdots \\ \cos \langle \boldsymbol{\omega}^Y, \mathbf{x} \rangle \\ \sin \langle \boldsymbol{\omega}^Y, \mathbf{x} \rangle \\ \end{array}\right], \qquad \Phi(\mathbf{x}) = \frac{1}{\sqrt{Y}} \left[ \begin{array}{c} \exp (+\langle \boldsymbol{\omega}^1, \mathbf{x} \rangle) \\ \exp (-\langle \boldsymbol{\omega}^1, \mathbf{x} \rangle) \\ \exp (+\langle \boldsymbol{\omega}^2, \mathbf{x} \rangle) \\ \exp (-\langle \boldsymbol{\omega}^2, \mathbf{x} \rangle) \\ \cdots \\ \exp (+\langle \boldsymbol{\omega}^Y, \mathbf{x} \rangle) \\ \exp (-\langle \boldsymbol{\omega}^Y, \mathbf{x} \rangle) \\ \end{array}\right], \quad \boldsymbol{\omega}^i \sim \mathcal{N}(0, \mathbf{I}_D), i \in [\![ Y ]\!]. \]
Rahimi A. and Recht B. 2007. Random Features for Large-scale Kernel Machines. Advances in Neural Information Processing Systems. [paper] Choromanski K., Likhosherstov V., Dohan D., Song X., Gane A., Sarlos T., Hawkins P., Davis J., Mohiuddin A., Kaiser L., Belanger D., Colwell, L. and Weller, A. 2021. Rethinking Attention with Performers. Proceedings of the International Conference on Learning Representations. [paper]
Note that both these random features generate \(2Y\)-dimensional feature map \(\Phi(\mathbf{x}) \in \mathbb{R}^{2Y}\) with \(Y\) random features \(\boldsymbol{\omega}^i, i \in [\![ Y ]\!]\). We implement the random features with trigonometric functions below.
def sin_cos_phi(
x: Float[Array, "... D"],
RF: Float[Array, "Y D"],
beta: float
) -> Float[Array, "... 2Y"]:
# print(x.shape, RF.shape)
"""Random features with trigonometric function"""
Y = RF.shape[0]
h = jnp.sqrt(beta) * (x @ RF.T)
return 1 / jnp.sqrt(Y) * jnp.concatenate(
[ jnp.cos(h), jnp.sin(h)], axis=-1
)Here we will visualize the quality of the kernel value approximations obtained with the trigonometric random features. We randomly generate samples \(\mathbf{x} \in \mathbb{R}^D\) from the domain, and compare their kernel values with the memories \(\boldsymbol{\xi}^\mu\). That is, we compare the true RBF kernel value \(\kappa(\sqrt{\beta}\mathbf{x}, \sqrt{\beta}\boldsymbol{\xi}^\mu) = \exp(-\frac{\beta}{2} \Vert \mathbf{x} - \boldsymbol{\xi}^\mu \Vert^2)\) to approximated kernel value using the random features \(\left\langle \Phi(\sqrt{\beta}\mathbf{x}), \sqrt{\beta}\Phi(\boldsymbol{\xi}^\mu) \right\rangle\) for different values of \(\beta\), \(\mu \in [\![ K ]\!]\) and different random samples \(\mathbf{x}\).
In the following, we compute the RBF kernel \(\kappa(\sqrt{\beta}\mathbf{x}, \sqrt{\beta}\boldsymbol{\xi}^\mu) = \exp(-\frac{\beta}{2} \Vert \mathbf{x} - \boldsymbol{\xi}^\mu \Vert^2)\).
We generate \(Y=2^{15}\) random features for the memories \(\boldsymbol{\Xi}\) using the sin_cos_phi function.
Now we generate NRANDS random \(\mathbf{x} \in \mathbb{R}^D\) and compare their exact and approximate kernel values with the memories.
rngidx = 9
NRANDS = 100
rxs = jr.uniform(
rnglist[rngidx], (NRANDS, D)
) * 2 * maxabs - maxabs
fig, axs = plt.subplots(
1, len(betas), figsize=(
len(betas) * figscaler,
figscaler
),
sharex=True, sharey=True
)
for b, ax in tqdm(
zip(betas, axs), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
phi_rxs = sin_cos_phi(rxs, RF, b)
true_kvals = np.array([rbfkernel(x, y, b) for x in rxs for y in Xi])
approx_kvals = np.array([x.dot(y) for x in phi_rxs for y in phi_Xi])
ax.scatter(true_kvals, approx_kvals)
ax.set_xlabel(r"$\kappa(\mathbf{x}, \boldsymbol{\xi}^\mu)$")
ax.set_title(r"$\beta$" + f":{b:0.2f}")
ax.axis('square')
axs[0].set_ylabel(r"$\langle \Phi(\mathbf{x}), \Phi(\boldsymbol{\xi}^\mu) \rangle$")
fig.tight_layout()
plt.show()100%|███████████████| 4/4 [00:00<00:00, 4.22it/s]

In the above figure, a good approximation would put all the points in the scatter plots on the diagonal. Based on these figures, we make the following observations with a fixed value of \(D, Y\): - Small values of \(\beta\) push the RBF kernel values close to 1, and the random features do not approximate these values well. - As \(\beta\) grows, the approximation quality improves, with \(\beta = 1\) producing really good approximation of the exact kernel values. - When \(\beta\) increases beyond a point, most pairwise RBF kernel values go close to zero, and approximation quality again falls.
Note that overall performance will continue to improve as the ratio \(D/Y\) decreases for any given value of \(\beta\).
Here we visualize the first NRFS=6 of the random features across the data domain for varying values of \(\beta\). The use of the trigonometric function is visible through the periodic nature of these random features, where larger values of \(\beta\) lead to shorter periods.
NRFS = 6
fig, axs = plt.subplots(
NRFS, len(betas), figsize=(
len(betas) * figscaler,
NRFS * figscaler
),
sharex=True, sharey=True
)
for bidx, b in tqdm(
enumerate(betas), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
rfs = np.zeros([2*NRFS, V.shape[1], V.shape[2]])
for i in range(nsteps):
rfs[:, i, :] = sin_cos_phi(V[:, i, :].T, RF[:NRFS, :], b).T
for i in range(NRFS):
plot_energy_landscape(
rfs[i, :, :], axs[i, bidx],
np.array([xmin, xmax, ymin, ymax]),
colormap='YlGn'
)
plot_states(Xi, axs[i, bidx], marker='*', color=MCOLOR)
axs[0, bidx].set_title(r"$\beta$" + f":{b:0.2f}")
# break
plt.show()100%|███████████████| 4/4 [00:00<00:00, 10.35it/s]

Given the random features \(\Phi: \mathbb{R}^D \to \mathbb{R}^{2Y}\), we can approximate the energy as \[\begin{align} E_\beta( \mathbf{v}; \boldsymbol{\Xi} ) & = - \frac{1}{\beta} \log \sum_{\mu = 1}^K \exp \left(- \frac{\beta}{2} \left\Vert \mathbf{v} - \boldsymbol{\xi}^\mu \right \Vert^2 \right) \\ & \approx \sum_{\mu = 1}^K \left\langle \Phi(\mathbf{v}), \Phi(\boldsymbol{\xi}^\mu) \right\rangle = \left\langle \Phi(\mathbf{v}), \sum_{\mu = 1}^K \Phi(\boldsymbol{\xi}^\mu) \right\rangle \\ & = \left\langle \Phi(\mathbf{v}), \mathbf{T} \right\rangle = \tilde{E}_\beta(\mathbf{v}, \mathbf{T}), \qquad \text{with} \quad \mathbf{T} = \sum_{\mu = 1}^K \Phi(\boldsymbol{\xi}^\mu). \end{align}\]
We implement this approximate energy below using random features below given the \(\{ \boldsymbol{\omega}^i, i \in [\![ Y ]\!] \}\) and the aggregated memories \(\mathbf{T} \in \mathbb{R}^{2Y}\).
Here we compare the exact energy landscape to energy landscape approximated with random features for varying values of \(\beta\) given the set of memories \(\boldsymbol{\Xi}\). For a given value of \(\beta\), we first compute \(\mathbf{T} = \sum_{\mu=1}^K \Phi(\sqrt{\beta} \boldsymbol{\xi}^\mu )\), and the use it to compute the approximate energy landscape.
fig, axs = plt.subplots(
2, len(betas), figsize=(
len(betas) * figscaler,
2 * figscaler
),
sharex=True, sharey=True
)
for bidx, b in tqdm(
enumerate(betas), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
# Computing the T tensor,
# summing the random features
# over all memories
T_Xi = sin_cos_phi(Xi, RF, b).sum(0)
# Computing the approx energy
# over the domain
app_en = np.zeros_like(V[0])
for i in range(nsteps):
app_en[i, :] = approx_lse_energy(
V[:, i, :].T, RF, b, T_Xi
)
# Plotting the exact and approx energy
plot_energy_landscape(
beta_en_map[b], axs[0, bidx],
np.array([xmin, xmax, ymin, ymax])
)
plot_states(
Xi, axs[0, bidx],
marker='*', color=MCOLOR
)
plot_energy_landscape(
app_en, axs[1, bidx],
np.array([xmin, xmax, ymin, ymax])
)
axs[0, bidx].set_title(r"$\beta$" + f":{b:0.2f}")
plt.show()100%|███████████████| 4/4 [00:02<00:00, 1.89it/s]

For small values of \(\beta\), the exact and approximate energy landscapes appear visually similar. However, for larger values of \(\beta\), the energy landscapes start differing significantly, especially farther away from the memories. However, note how the approximate energy still forms a local minima around each of the original memories even though the actual basins of attraction of this approximate energy are significantly smaller.
For a state vector \(\mathbf{v} \in \mathbb{R}^D\), we can approximately reduce its energy \(E_\beta(\mathbf{v}; \boldsymbol{\Xi})\) by utilizing the gradient \(\nabla_{\mathbf{v}} \tilde{E}_\beta(\mathbf{v}; \mathbf{T})\) of the random-feature based approximate energy \(\tilde{E}_\beta(\mathbf{v}; \mathbf{T})\). Initializing the energy descent at the input \(\mathbf{q}\), that is \(\tilde{\mathbf{v}}^{(0)} \gets \mathbf{q}\), we perform the following gradient descent steps for \(T\) iterations: \[ \tilde{\mathbf{v}}^{(t)} \gets \tilde{\mathbf{v}}^{(t-1)} - \alpha \nabla_{\mathbf{v}} \tilde{E}_\beta( \tilde{\mathbf{v}}^{(t-1)}; \mathbf{T} ), \] for \(t = 1, \ldots, T\) with \(\alpha > 0\) as the step-size (or learning rate) for the energy descent. The final \(\tilde{\mathbf{v}}^{(T)}\) is the output of this model. This output will be different than the output \(\mathbf{v}^{(T)}\) obtained using the exact energy gradient \(\nabla_{\mathbf{v}} E_\beta( \mathbf{v}; \boldsymbol{\Xi} )\).
The gradient of the approximate energy does not require access to the original memories \(\boldsymbol{\Xi}\), and can be computed solely using the random features \(\{ \boldsymbol{\omega}^i, i \in [\![ Y ]\!] \}\) and the consolidated memories \(\mathbf{T} \in \mathbb{R}^{2Y}\): \[ \nabla_{\mathbf{v}} \tilde{E}_\beta ( \mathbf{v}; \mathbf{T} ) = - \frac{1}{\beta} \nabla_{\mathbf{v}} \log \left\langle \Phi(\mathbf{v}), \mathbf{T} \right\rangle = - \frac{1}{\beta \left\langle \Phi(\mathbf{v}), \mathbf{T} \right\rangle} \left[ \nabla_{\mathbf{v}} \Phi(\mathbf{v}) \right]^\top \mathbf{T}. \] The gradient \(\nabla_{\mathbf{v}} \Phi(\mathbf{v})\) of the random feature map \(\Phi: \mathbb{R}^D \to \mathbb{R}^{2Y}\) with respect to its input is a \((2Y \times D)\) matrix.
We can implement this using auto-differentiation in JAX by computing the gradient of the approx_lse_energy function with respect to its input state.
def approx_lse_energy_descent(
q: Float[Array, "D"],
RF: Float[Array, "Y D"],
beta: float,
T: Float[Array, "2Y"],
energy_fn,
depth: int=10,
alpha: float = 0.01,
return_grads=False,
clamp_idxs: Optional[Bool[Array, "D"]]=None
) -> Float[Array, "D"]:
"""Using the approx random feature energy"""
""" run energy descent"""
dEdxf = jax.jit(jax.value_and_grad(energy_fn))
logs = {}
@jax.jit
def step(x, i):
E, dEdx = dEdxf(x, RF, beta, T)
if clamp_idxs is not None:
dEdx = jnp.where(clamp_idxs, 0, dEdx)
x = x - alpha * dEdx
aux = (E, dEdx) if return_grads else (E,)
return x, aux
x, aux = jax.lax.scan(step, q, jnp.arange(depth))
logs['energies'] = aux[0]
if return_grads:
logs['grads'] = aux[1]
return x, logsWe will now compare the energy descent dynamics of this gradient using the random-feature based approximate energy to the exact energy gradient from earlier. We keep the step-size \(\alpha\) and the number of DenseAM layers \(T\) the same as the exact energy descent with \(\alpha = 0.01\) and \(T = 1000\). For each of the intermediate states obtained with this approximate energy descent, we compute the exact energy to check how it decreases through the DenseAM layers.
rngidx = 8
fig, axs = plt.subplots(
NQUERIES+1, len(betas), figsize=(
len(betas) * figscaler,
NQUERIES+2 * figscaler
),
sharex="row",
)
for bidx, b in tqdm(
enumerate(betas), total=len(betas),
colour=TQDMCOLOR, ncols=50
):
# using cached energy landscape
plot_energy_landscape(
beta_en_map[b], axs[0, bidx],
np.array([xmin, xmax, ymin, ymax])
)
plot_states(
Xi, axs[0, bidx],
marker='*', color=MCOLOR
)
axs[0, bidx].set_title(r"$\beta$" + f":{b:0.2f}")
# Computing the T tensor,
# summing the random features
# over all memories
T_Xi = sin_cos_phi(Xi, RF, b).sum(0)
# Randomly generating a query
queries = jr.uniform(
rnglist[rngidx], (NQUERIES, D)
) * 2 * maxabs - maxabs
beta_cache = []
for qidx, query in enumerate(queries):
qstates = [query]
qens = []
# Perform energy descent using approx gradient
for i in range(NSTATES):
query, logs = approx_lse_energy_descent(
query, RF, b, T_Xi, approx_lse_energy,
depth=NUPDATES, alpha=ALPHA
)
qstates += [query]
qens += [logs['energies']]
# using cached exact descent stats
ex_qstates, ex_qens = beta_true_states_en_map[b][qidx]
plot_states(
ex_qstates, axs[0, bidx],
marker='o', color=QCOLOR
)
plot_energy_descent(
ex_qens, axs[qidx+1, bidx], color=QCOLOR
)
qstates = np.array(qstates)
plot_states(
qstates, axs[0, bidx],
marker='.', color=AQCOLOR
)
qens = np.array([
[lse_energy(qs, Xi, b)]*NUPDATES
for qs in qstates
]).reshape(-1)
plot_energy_descent(
qens, axs[qidx+1, bidx], color=AQCOLOR
)
fig.tight_layout()
plt.show()100%|███████████████| 4/4 [00:11<00:00, 2.88s/it]

The above results show that, for small to moderately large \(\beta\), with sufficiently large number of random features \(Y\), the gradient of the random-feature based approximate energy matches the dynamics of the exact energy gradient. However, for large \(\beta\), the gradient of the approximate energy is no longer able to reduce the energy of the initial state if the initial state happens to be quite far from all the memories, implying a large initial energy \(E_\beta ( \mathbf{v}^{(0)}; \boldsymbol{\Xi} )\).
Our previous results showed that the random-feature based kernel approximation does not perform well if \(\beta\) is too small, or too large. However, the approximate kernel based energy gradient is sufficient for low \(\beta\) regime, highlighting that energy descent is possible even if the kernel approximation is poor. This is because the energy of the initial state at low \(\beta\) is already quite low.
--------------------------------------------------------------------------- AssertionError Traceback (most recent call last) Cell In[21], line 1 ----> 1 assert False AssertionError:
class DrDAM:
"""Defines the interface and basic methods for all KernelizableDAMs"""
def __init__(self, key, D, Y, beta):
self.RF = jr.normal(key, (Y, D))
self.beta = beta
self.Y = Y
self.Tdim = 2*Y
self.D = D
def phi(self, x: Float[Array, "... D"]) -> Float[Array, "... 2Y"]:
"""Compute the basis function """
return sin_cos_phi(x, self.RF, self.beta)
def sim(self, x: Float[Array, "D"], y: Float[Array, "D"]) -> Float[Array, ""]:
"""Compute the standard L2 similarity between two vectors."""
return jnp.exp(-self.beta / 2 * ((x - y) ** 2).sum())
def energy(
self, x: Float[Array, "D"], memories: Float[Array, "M D"]
) -> Float[Array, ""]:
"""Compute the standard L2 energy"""
return -(1 / self.beta) * jax.nn.logsumexp(
-self.beta / 2 * ((x - memories) ** 2).sum(-1), axis=0
)
def kernel_energy(
self, x: Float[Array, "D"], T: Float[Array, "2Y"], eps=1e-10
) -> Float[Array, ""]:
"""Compute the approximate kernelized energy"""
h = self.phi(x) @ T
h = jnp.clip(h, a_min=eps)
return -(1 / self.beta) * jnp.log(h)
def kernel_sim(
self, x: Float[Array, "D"], y: Float[Array, "D"]
) -> Float[Array, ""]:
"""Compute the approximate kernel similarity of `x` and `y`"""
return self.phi(x) @ self.phi(y)
def kernelize_memories(self, memories: Float[Array, "M D"], **kwargs) -> Float[Array, "2Y"]:
"""
Naive implementation that BLOWS UP with many memories `n`,
since it creates the entire memory matrix from scratch
"""
return self.phi(memories).sum(0)
def recall(
self, q: Float[Array, "D"], memories: Float[Array, "M D"],
depth: int=1000, alpha: float = 0.1, return_grads=False,
clamp_idxs: Optional[Bool[Array, "D"]]=None
) -> Float[Array, "D"]:
"""Using the normal similarity function, run energy dynamics"""
dEdxf = jax.jit(jax.value_and_grad(self.energy))
logs = {}
def step(x, i):
E, dEdx = dEdxf(x, memories)
if clamp_idxs is not None:
dEdx = jnp.where(clamp_idxs, 0, dEdx)
x = x - alpha * dEdx
aux = (E, dEdx) if return_grads else (E,)
return x, aux
x, aux = jax.lax.scan(step, q, jnp.arange(depth))
logs['energies'] = aux[0]
if return_grads:
logs['grads'] = aux[1]
return x, logs
def kernel_recall(
self, q: Float[Array, "D"], T: Float[Array, "2Y"],
depth: int=1000, alpha: float = 0.1, return_grads=False,
clamp_idxs: Optional[Bool[Array, "D"]]=None
) -> Float[Array, "D"]:
"""Using the kernelized similarity function, run energy dynamics"""
dEdxf = jax.jit(jax.value_and_grad(self.kernel_energy))
logs = {}
@jax.jit
def step(x, i):
E, dEdx = dEdxf(x, T)
if clamp_idxs is not None:
dEdx = jnp.where(clamp_idxs, 0, dEdx)
x = x - alpha * dEdx
aux = (E, dEdx) if return_grads else (E,)
return x, aux
x, aux = jax.lax.scan(step, q, jnp.arange(depth))
logs['energies'] = aux[0]
if return_grads:
logs['grads'] = aux[1]
return x, logs