Butanium commited on
Commit
51864ca
·
verified ·
1 Parent(s): d21e423

Added crosscoder config

Browse files
Files changed (1) hide show
  1. config.json +1 -0
config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"model_type": "crosscoder", "model_0": "Qwen2.5-0.5B", "model_1": "Qwen2.5-0.5B-Instruct", "activation_dim": 896, "dict_size": 32000, "num_layers": 2, "mu": 0.05, "learning_rate": 0.0001, "batch_size": 1024, "dataset": "lmsys/lmsys-chat-1m", "num_tokens": 50000896, "width": 32000, "L1_penalty": 0.05, "l0_validation": 73, "frac_var_explained_validation": 0.84, "dead_latents_validation": 1050}