diagrams/assets/sparse-autoencoder/sparse-autoencoder.typ at main · janosh/diagrams · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#import "@preview/cetz:0.5.2": canvas, draw
#import draw: circle, on-layer
#import "../_shared/network.typ": fully-connect
#import "@preview/suiji:0.5.1"

#set page(width: auto, height: auto, margin: 8pt, fill: none)

#canvas({
  // Layer configuration
  let (n-input-neurons, n-hidden-neurons) = (6, 12)
  let (input-len, hidden-len) = (20, 30)
  let sparsity = 0.3

  // Generate activations
  let input-rng = suiji.gen-rng(43)
  let (input-rng, input-activations) = suiji.uniform(input-rng, size: n-input-neurons)

  let hidden-rng = suiji.gen-rng(47)
  let (hidden-rng, uniform-values) = suiji.uniform(hidden-rng, size: n-hidden-neurons)
  let (hidden-rng, hidden-activations) = suiji.uniform(hidden-rng, size: n-hidden-neurons)
  let is-alive = uniform-values.map(x => x < sparsity)

  on-layer(1, {
    // Input layer
    for ii in range(n-input-neurons) {
      circle(
        (0, ii * input-len / (n-input-neurons - 1)),
        radius: 1,
        fill: blue.transparentize(input-activations.at(1 - ii) * 100%),
        stroke: blue.darken(30%) + 2pt,
        name: "in-" + str(ii),
      )
    }
    // Hidden layer
    for ii in range(n-hidden-neurons) {
      let y-offset = (input-len - hidden-len) / 2
      circle(
        (10, ii * hidden-len / (n-hidden-neurons - 1) + y-offset),
        radius: 1,
        fill: green.transparentize(
          100% * (1 - float(is-alive.at(ii)) * hidden-activations.at(ii)),
        ),
        stroke: green.darken(30%) + 2pt,
        name: "hidden-" + str(ii),
      )
    }
    // Output layer
    for ii in range(n-input-neurons) {
      circle(
        (20, ii * input-len / (n-input-neurons - 1)),
        radius: 1,
        fill: blue.transparentize(input-activations.at(1 - ii) * 100%),
        stroke: blue.darken(30%) + 2pt,
        name: "out-" + str(ii),
      )
    }
  })

  on-layer(0, {
    let wire = fully-connect.with(start: 0, stroke: gray + 1.5pt)
    wire("in-", "hidden-", n-input-neurons, n-hidden-neurons)
    wire("hidden-", "out-", n-hidden-neurons, n-input-neurons)
  })
})