Training Reservoir Computing Models
Training an RC model means fitting the readout. The default objective is ridge regression; other linear and SVM objectives are available through extensions.
Ridge regression
using ReservoirComputing
using LuxCore: setup
using Random
Random.seed!(42)
rng = MersenneTwister(42)
input_data = rand(Float32, 3, 100)
target_data = rand(Float32, 5, 100)
model = ESN(3, 100, 5)
ps, st = setup(rng, model)
ps, st = train(model, input_data, target_data, ps, st;
objective = RidgeRegression(),
solver = QRFactorization())((reservoir = (input_matrix = Float32[0.086408615 0.0998888 -0.013271046; -0.06170206 -0.06359978 0.0921186; … ; -0.027093245 -0.04336114 -0.04771917; 0.07098396 0.015740562 0.019439578], reservoir_matrix = Float32[0.0 0.0 … 0.0045833415 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (weight = [-653.597492490338 -247.25807418310598 … 126.37102722607627 663.6413997356498; -486.3743086285802 38.02180496645455 … -77.22950170183633 -99.6384323676058; … ; 396.2246665195202 -86.8648959077094 … 26.078160022515345 -405.2525292694728; -535.9488971657303 87.67290197939872 … -94.1879512210102 -157.435352595814],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 23562, 10020, 692, 22308, 882)),), carry = (Float32[0.095440336; -0.010096364; … ; -0.17061083; -0.22067434;;],)), state_modifiers = (), readout = NamedTuple()))objective chooses what to fit (here ridge). solver chooses how to solve it; omitting solver uses QRFactorization.
ps, st = train(model, input_data, target_data, ps, st;
objective = RidgeRegression())((reservoir = (input_matrix = Float32[0.086408615 0.0998888 -0.013271046; -0.06170206 -0.06359978 0.0921186; … ; -0.027093245 -0.04336114 -0.04771917; 0.07098396 0.015740562 0.019439578], reservoir_matrix = Float32[0.0 0.0 … 0.0045833415 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (weight = [4480.000296004649 928.8349000207866 … -2579.313279730249 -3101.8880251303344; -23639.40643600179 -3838.8241781194524 … 11076.115526963154 13031.279997790645; … ; -8218.735898864483 -1783.603703256959 … 4191.210905979773 4693.319981326809; -799.4328445187732 -210.44631000273253 … 490.23567335584136 172.56756042242426],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 23562, 10020, 692, 22308, 882)),), carry = (Float32[0.09544154; -0.010095287; … ; -0.17061086; -0.22067615;;],)), state_modifiers = (), readout = NamedTuple()))Changing the ridge solver
Other LinearSolve.jl algorithms:
using LinearSolve
ps, st = train(model, input_data, target_data, ps, st;
objective = RidgeRegression(),
solver = SVDFactorization())((reservoir = (input_matrix = Float32[0.086408615 0.0998888 -0.013271046; -0.06170206 -0.06359978 0.0921186; … ; -0.027093245 -0.04336114 -0.04771917; 0.07098396 0.015740562 0.019439578], reservoir_matrix = Float32[0.0 0.0 … 0.0045833415 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (weight = [4473.8306528394405 927.9582944074056 … -2576.664820289712 -3098.785253117207; -23621.202117858928 -3836.6164810493683 … 11068.556148978572 13022.93805098243; … ; -8209.137821165012 -1782.2516372812802 … 4187.023127804454 4688.559009231197; -799.8902543512833 -210.566141127396 … 490.4831398143892 172.92740710286702],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 23562, 10020, 692, 22308, 882)),), carry = (Float32[0.09544154; -0.010095247; … ; -0.17061083; -0.22067612;;],)), state_modifiers = (), readout = NamedTuple()))Legacy built-in path:
ps, st = train(model, input_data, target_data, ps, st;
objective = RidgeRegression(),
solver = QRSolver())((reservoir = (input_matrix = Float32[0.086408615 0.0998888 -0.013271046; -0.06170206 -0.06359978 0.0921186; … ; -0.027093245 -0.04336114 -0.04771917; 0.07098396 0.015740562 0.019439578], reservoir_matrix = Float32[0.0 0.0 … 0.0045833415 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (weight = [4488.217692150659 930.4221019701693 … -2583.583808731632 -3107.1537825008327; -23658.39569858292 -3842.7885679180913 … 11086.596373107044 13044.800414781374; … ; -8226.620955579885 -1785.202358358723 … 4195.454055422987 4698.830399258094; -798.6199502559307 -210.31949165384435 … 489.8735718021248 172.20555359595627],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 23562, 10020, 692, 22308, 882)),), carry = (Float32[0.09544155; -0.010095281; … ; -0.17061086; -0.22067615;;],)), state_modifiers = (), readout = NamedTuple()))See LinearSolve's solver list.
Other linear objectives
MLJLinearModels.jl provides additional regressors (lasso, elastic net, …).
using MLJLinearModels
ps, st = train(model, input_data, target_data, ps, st;
objective = LassoRegression(fit_intercept = false),
solver = ProxGrad())((reservoir = (input_matrix = Float32[0.086408615 0.0998888 -0.013271046; -0.06170206 -0.06359978 0.0921186; … ; -0.027093245 -0.04336114 -0.04771917; 0.07098396 0.015740562 0.019439578], reservoir_matrix = Float32[0.0 0.0 … 0.0045833415 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (weight = Float32[0.0 0.0 … -0.0 -0.0; 0.0 0.0 … -0.0 -0.0; … ; 0.0 0.0 … -0.0 -0.0; 0.0 0.0 … -0.0 -0.0],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 23562, 10020, 692, 22308, 882)),), carry = (Float32[0.09544155; -0.010095294; … ; -0.17061086; -0.22067615;;],)), state_modifiers = (), readout = NamedTuple()))See MLJLinearModels models and solvers. Not every solver works with every model. MLJ also exports a type named RidgeRegression; write MLJLinearModels.RidgeRegression when both packages are loaded.
Extending readout fitting
Package extensions can support another training objective by defining:
ReservoirComputing.__fit_readout(
objective::MyObjective,
states::AbstractMatrix,
target_data::AbstractMatrix;
solver = nothing,
kwargs...,
)The columns of states and target_data are aligned training samples. A matrix-valued result must have size (n_outputs, n_features) and is installed in a standard linear readout. Backends that return another fitted object must also implement ReservoirComputing.addreadout! for compatible model and readout types.
The model-level train function omits the solver keyword when the user supplies solver=nothing; otherwise it forwards the solver together with any additional keywords. Extension methods should reject unsupported solvers and keywords with an ArgumentError.
__fit_readout is an internal extension interface and is not a user-facing training entry point. Its compatibility is not guaranteed before version 1.0. Users should always call the model-level train API.
Support Vector Regression
ReservoirComputing.jl also allows users to train RC models with support vector regression through LIBSVM.jl. However, the majority of builtin models in the library uses a LinearReadout by default, which can only be trained with linear regression. In order to use support vector regression, one needs to build a model with SVMReadout
using LIBSVM
model = ReservoirComputer(
StatefulLayer(ESNCell(3=>100)),
SVMReadout(100=>5)
)
ps, st = setup(rng, model)((reservoir = (input_matrix = Float32[-0.08695352 0.065400176 0.034358095; 0.062833786 0.07213054 -0.043287825; … ; -0.08326278 -0.018158173 -0.027971745; 0.0061006784 0.09752395 -0.03747182], reservoir_matrix = Float32[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = NamedTuple()), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 46122, 32580, 920, 44868, 762)),), carry = nothing), state_modifiers = (), readout = NamedTuple()))We can now train our new model similarly to before:
ps, st = train(model, input_data, target_data, ps, st;
objective = EpsilonSVR()) # from LIBSVM((reservoir = (input_matrix = Float32[-0.08695352 0.065400176 0.034358095; 0.062833786 0.07213054 -0.043287825; … ; -0.08326278 -0.018158173 -0.027971745; 0.0061006784 0.09752395 -0.03747182], reservoir_matrix = Float32[0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0; … ; 0.0 0.0 … 0.0 0.0; 0.0 0.0 … 0.0 0.0]), state_modifiers = (), readout = (models = Any[LIBSVM.EpsilonSVR(LIBSVM.Kernel.RadialBasis, 0.01, 0.1, 1.0, 3, 0.0, 0.001, true, false, LIBSVM.SVM{Float64, LIBSVM.Kernel.KERNEL}(LIBSVM.EpsilonSVR, LIBSVM.Kernel.RadialBasis, nothing, 100, 100, 2, Float64[], Int32[], Float64[], Int32[], LIBSVM.SupportVectors{Vector{Float32}, Matrix{Float32}}(0, Int32[], Float32[], Matrix{Float32}(undef, 100, 0), Int32[], LIBSVM.SVMNode[]), 0.0, Matrix{Float64}(undef, 0, 1), Float64[], Float64[], [-0.0036369473622437157], 3, 0.01, 200.0, 0.001, 1.0, 0.5, 0.1, true, false)), LIBSVM.EpsilonSVR(LIBSVM.Kernel.RadialBasis, 0.01, 0.1, 1.0, 3, 0.0, 0.001, true, false, LIBSVM.SVM{Float64, LIBSVM.Kernel.KERNEL}(LIBSVM.EpsilonSVR, LIBSVM.Kernel.RadialBasis, nothing, 100, 100, 2, Float64[], Int32[], Float64[], Int32[], LIBSVM.SupportVectors{Vector{Float32}, Matrix{Float32}}(0, Int32[], Float32[], Matrix{Float32}(undef, 100, 0), Int32[], LIBSVM.SVMNode[]), 0.0, Matrix{Float64}(undef, 0, 1), Float64[], Float64[], [-0.0036369473622437157], 3, 0.01, 200.0, 0.001, 1.0, 0.5, 0.1, true, false)), LIBSVM.EpsilonSVR(LIBSVM.Kernel.RadialBasis, 0.01, 0.1, 1.0, 3, 0.0, 0.001, true, false, LIBSVM.SVM{Float64, LIBSVM.Kernel.KERNEL}(LIBSVM.EpsilonSVR, LIBSVM.Kernel.RadialBasis, nothing, 100, 100, 2, Float64[], Int32[], Float64[], Int32[], LIBSVM.SupportVectors{Vector{Float32}, Matrix{Float32}}(0, Int32[], Float32[], Matrix{Float32}(undef, 100, 0), Int32[], LIBSVM.SVMNode[]), 0.0, Matrix{Float64}(undef, 0, 1), Float64[], Float64[], [-0.0036369473622437157], 3, 0.01, 200.0, 0.001, 1.0, 0.5, 0.1, true, false)), LIBSVM.EpsilonSVR(LIBSVM.Kernel.RadialBasis, 0.01, 0.1, 1.0, 3, 0.0, 0.001, true, false, LIBSVM.SVM{Float64, LIBSVM.Kernel.KERNEL}(LIBSVM.EpsilonSVR, LIBSVM.Kernel.RadialBasis, nothing, 100, 100, 2, Float64[], Int32[], Float64[], Int32[], LIBSVM.SupportVectors{Vector{Float32}, Matrix{Float32}}(0, Int32[], Float32[], Matrix{Float32}(undef, 100, 0), Int32[], LIBSVM.SVMNode[]), 0.0, Matrix{Float64}(undef, 0, 1), Float64[], Float64[], [-0.0036369473622437157], 3, 0.01, 200.0, 0.001, 1.0, 0.5, 0.1, true, false)), LIBSVM.EpsilonSVR(LIBSVM.Kernel.RadialBasis, 0.01, 0.1, 1.0, 3, 0.0, 0.001, true, false, LIBSVM.SVM{Float64, LIBSVM.Kernel.KERNEL}(LIBSVM.EpsilonSVR, LIBSVM.Kernel.RadialBasis, nothing, 100, 100, 2, Float64[], Int32[], Float64[], Int32[], LIBSVM.SupportVectors{Vector{Float32}, Matrix{Float32}}(0, Int32[], Float32[], Matrix{Float32}(undef, 100, 0), Int32[], LIBSVM.SVMNode[]), 0.0, Matrix{Float64}(undef, 0, 1), Float64[], Float64[], [-0.0036369473622437157], 3, 0.01, 200.0, 0.001, 1.0, 0.5, 0.1, true, false))],)), (reservoir = (cell = (rng = Random.MersenneTwister(42, (0, 47124, 46122, 18, 44868, 762)),), carry = (Float32[-0.011674796; 0.32498732; … ; -0.13789222; -0.009694248;;],)), state_modifiers = (), readout = NamedTuple()))