Hello,
I was re implementing some python code that I had and I realized that it was much slower in julia.
The two functions I am comparing are written below and can be found here (https://github.com/davidbp/learn_julia/tree/master/speed_tests).
Python code takes 0.4 seconds and julia code 1.8 or 3.5 depending if I use the standard product or a kron (I also do not understand the huge speed difference about these two functions).
The code is not fully vectorized and I was expecting it to be faster in julia than in python. The fully vectorized version in python is even faster.
The code in python and julia is essentially a translation but the number of allocations and gc time in the Julia code suggest I am doing something wrong.
python rbm.py
— 0.41869401931762695 seconds —
julia rbm.jl
This RBM used scalar product
1.811265 seconds (580.21 k allocations: 1.631 GB, 16.26% gc time)
This RBM used kron product
3.495652 seconds (2.62 M allocations: 1.673 GB, 4.42% gc time)
##### Julia function
function contrastive_divergence_col_K(Xbatch, rbm, K::Int64, lr::Float64)
batch_size = size(Xbatch)[2]
Delta_W = zeros(size(rbm.W))
Delta_b = zeros(size(rbm.vis_bias))
Delta_c = zeros(size(rbm.hid_bias))
for i in 1:batch_size
x = Xbatch[:,i]
xneg = Xbatch[:,i]
for k in 1:K
hneg = sigmoid( rbm.W * xneg .+ rbm.hid_bias) .> rand(rbm.n_hid)
xneg = sigmoid( rbm.W' * hneg .+ rbm.vis_bias) .> rand(rbm.n_vis)
end
ehp = sigmoid(rbm.W * x + rbm.hid_bias)
ehn = sigmoid(rbm.W * xneg + rbm.hid_bias)
### kron vs no kron???
#Delta_W += lr * ( x * ehp' - xneg * ehn')'
Delta_W += lr * (kron(x, ehp') - kron(xneg, ehn'))'
Delta_b += lr * (x - xneg)
Delta_c += lr * (ehp - ehn)
end
rbm.W += Delta_W / batch_size;
rbm.vis_bias += Delta_b / batch_size;
rbm.hid_bias += Delta_c / batch_size;
return
##### Python function
def contrastive_divergence_rows_K(Xbatch, rbm, K, lr):
batch_size = Xbatch.shape[0]
Delta_W = np.zeros(rbm.W.shape)
Delta_b = np.zeros([rbm.n_vis,1])
Delta_c = np.zeros([rbm.n_hid,1])
for i in range(batch_size):
x = Xbatch[i:i+1,:]
xneg = Xbatch[i:i+1,:]
for k in range(K):
hneg = sigmoid( np.dot(xneg,rbm.W) + rbm.hid_bias.T) > np.random.rand(1,rbm.n_hid)
xneg = sigmoid( np.dot(hneg,rbm.W.T) + rbm.vis_bias.T) > np.random.rand(1,rbm.n_vis)
ehp = sigmoid( np.dot(x,rbm.W) + rbm.hid_bias.T)
ehn = sigmoid( np.dot(xneg,rbm.W) + rbm.hid_bias.T)
Delta_W += lr * ( x * ehp.T - xneg * ehn.T).T
Delta_b += lr * (x - xneg).T
Delta_c += lr * (ehp - ehn).T
rbm.W += Delta_W / batch_size;
rbm.vis_bias += Delta_b / batch_size;
rbm.hid_bias += Delta_c / batch_size;
return