NVIDIA CUDA: training modes

github.com/openfluke/welvet/nativecudalinux/windows amd64

Why it exists

Keep repeated training updates in the mounted CUDA session.

What it is

Train two Dense layers using one of the 29 numbered modes, with persistent Step Line state and real Grid layouts for Mesh modes.

Training families

-mode all exercises all 29 modes. Step modes retain their Line session across calls. Mesh modes use a three-cell Grid with disabled ingress and two active layers, advancing four ticks per call. Bundled welvet.cuda 0.79 covers train modes through StepTweenSplitSparse. These small demonstrations validate usage; they are not performance measurements.

go run . -mode NormalBP -steps 8
go run . -mode all -steps 8

Explicit mounted sessions keep updated weights native until SyncToGo(). Close discards any updates not returned to Go. See chapter 92 before saving.

Requirements: chapter 89.

Go example

examples/91-nvidia-training/main.go

Run:cd welvet/examples/91-nvidia-training && source ../env.sh && go run .

// Native CUDA MSE training: persistent Stack Line state or an actual Grid clock.
package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"

	"github.com/openfluke/welvet/architecture"
	"github.com/openfluke/welvet/core"
	"github.com/openfluke/welvet/layers/dense"
	"github.com/openfluke/welvet/layers/parallel"
	"github.com/openfluke/welvet/nativecuda"
	"github.com/openfluke/welvet/quant"
)

func main() {
	if e := run(); e != nil {
		log.Fatal(e)
	}
}

func run() error {
	selected := flag.String("mode", "NormalBP", "training mode name, or all 29")
	steps := flag.Int("steps", 8, "training calls; retain the session for Step pipeline state")
	flag.Parse()
	if *steps < 1 {
		return fmt.Errorf("steps must be positive")
	}
	if !cudaPlatformOK() {
		fmt.Println("SKIP: NVIDIA CUDA needs linux/amd64 or windows/amd64 and cgo")
		return nil
	}
	if *selected == "all" {
		for mode := parallel.ModeNormalBP; mode <= parallel.ModeStepTweenSplitSparse; mode++ {
			if e := train(mode, *steps); e != nil {
				if mode == parallel.ModeNormalBP {
					fmt.Printf("SKIP: CUDA unavailable (%v)\n", e)
					return nil
				}
				return e
			}
		}
		return nil
	}
	mode, e := parallel.ParseTrainMode(*selected)
	if e != nil {
		return e
	}
	if mode < parallel.ModeNormalBP || mode > parallel.ModeStepTweenSplitSparse {
		return fmt.Errorf("choose one of the 29 update modes")
	}
	if e := train(mode, *steps); e != nil {
		fmt.Printf("SKIP: CUDA unavailable (%v)\n", e)
		return nil
	}
	return nil
}

func train(mode parallel.TrainMode, steps int) error {
	var ops []any
	for layer := 0; layer < 2; layer++ {
		w := make([]float32, 16)
		for i := range w {
			w[i] = float32(i%5-2) * .04
		}
		l, e := dense.NewConfigured(4, 4, core.ActivationTanh, core.DTypeFloat32, quant.FormatNone, w)
		if e != nil {
			return e
		}
		ops = append(ops, l)
	}
	stack, e := parallel.NewStack(ops...)
	if e != nil {
		return e
	}
	var model any = stack
	schedule := "Stack"
	if mode.RequiresGrid() {
		g := architecture.NewGrid(3, 1, 1, 1)
		g.Cells[0].Layer.IsDisabled = true
		for i, op := range ops {
			g.Cells[i+1].Layer = op.(*dense.Layer).Core
			g.Cells[i+1].Op = op
		}
		model = g
		schedule = "Grid mesh clock (4 ticks/call)"
	} else if mode.IsLineStep() {
		schedule = "Stack Line pipeline"
	}
	s, e := nativecuda.Mount(model)
	if e != nil {
		return e
	}
	defer s.Close()
	x := core.NewTensor[float32](2, 4)
	for i := range x.Data {
		x.Data[i] = float32(i%5-2) * .1
	}
	target := core.NewTensor[float32](2, 4)
	var loss float64
	for i := 0; i < steps; i++ {
		loss, e = s.TrainMSE(x, target, mode, .01, 4)
		if e != nil {
			return fmt.Errorf("%s: %w", mode, e)
		}
	}
	if e = s.SyncToGo(); e != nil {
		return e
	}
	fmt.Printf("mode=%s schedule=%s calls=%d loss=%.8g weights=synced-to-Go\n", mode, schedule, steps, loss)
	return nil
}

func cudaPlatformOK() bool {
	return runtime.GOARCH == "amd64" && (runtime.GOOS == "linux" || runtime.GOOS == "windows")
}

Output

exit 0 · last run via go run .

mode=NormalBP schedule=Stack calls=8 loss=6.7954462e-06 weights=synced-to-Go

Validate (harness)

cd welvet/examples/91-nvidia-training && source ../env.sh && go run .