91. NVIDIA CUDA: training modes

Part: XIII · NVIDIA acceleration (welvet.cuda)
Package: nativecuda + parallel + architecture
Status: ok: needs Linux/Windows amd64 + NVIDIA runtime

When

Train a mounted model using native CUDA credit and optimizer kernels.

Where

go run .
go run . -mode StepBP -steps 8
go run . -mode MeshTweenChain
go run . -mode all

Why

Keep one session alive for Step pipeline state; use an actual Grid for Mesh modes.

What

Runs a two-layer Dense model in NormalBP by default. All 29 update modes are selectable. Mesh modes use a disabled ingress plus two active Grid cells and four clock ticks per call.

Requirements

Same as 89-nvidia-cuda. Bundled engine welvet.cuda 0.79 covers train modes through StepTweenSplitSparse.

Captured sample

mode=NormalBP schedule=Stack calls=8 loss=… weights=synced-to-Go

main.go

Download source ↓

// Native CUDA MSE training: persistent Stack Line state or an actual Grid clock.
package main

import (
	"flag"
	"fmt"
	"log"
	"runtime"

	"github.com/openfluke/welvet/architecture"
	"github.com/openfluke/welvet/core"
	"github.com/openfluke/welvet/layers/dense"
	"github.com/openfluke/welvet/layers/parallel"
	"github.com/openfluke/welvet/nativecuda"
	"github.com/openfluke/welvet/quant"
)

func main() {
	if e := run(); e != nil {
		log.Fatal(e)
	}
}

func run() error {
	selected := flag.String("mode", "NormalBP", "training mode name, or all 29")
	steps := flag.Int("steps", 8, "training calls; retain the session for Step pipeline state")
	flag.Parse()
	if *steps < 1 {
		return fmt.Errorf("steps must be positive")
	}
	if !cudaPlatformOK() {
		fmt.Println("SKIP: NVIDIA CUDA needs linux/amd64 or windows/amd64 and cgo")
		return nil
	}
	if *selected == "all" {
		for mode := parallel.ModeNormalBP; mode <= parallel.ModeStepTweenSplitSparse; mode++ {
			if e := train(mode, *steps); e != nil {
				if mode == parallel.ModeNormalBP {
					fmt.Printf("SKIP: CUDA unavailable (%v)\n", e)
					return nil
				}
				return e
			}
		}
		return nil
	}
	mode, e := parallel.ParseTrainMode(*selected)
	if e != nil {
		return e
	}
	if mode < parallel.ModeNormalBP || mode > parallel.ModeStepTweenSplitSparse {
		return fmt.Errorf("choose one of the 29 update modes")
	}
	if e := train(mode, *steps); e != nil {
		fmt.Printf("SKIP: CUDA unavailable (%v)\n", e)
		return nil
	}
	return nil
}

func train(mode parallel.TrainMode, steps int) error {
	var ops []any
	for layer := 0; layer < 2; layer++ {
		w := make([]float32, 16)
		for i := range w {
			w[i] = float32(i%5-2) * .04
		}
		l, e := dense.NewConfigured(4, 4, core.ActivationTanh, core.DTypeFloat32, quant.FormatNone, w)
		if e != nil {
			return e
		}
		ops = append(ops, l)
	}
	stack, e := parallel.NewStack(ops...)
	if e != nil {
		return e
	}
	var model any = stack
	schedule := "Stack"
	if mode.RequiresGrid() {
		g := architecture.NewGrid(3, 1, 1, 1)
		g.Cells[0].Layer.IsDisabled = true
		for i, op := range ops {
			g.Cells[i+1].Layer = op.(*dense.Layer).Core
			g.Cells[i+1].Op = op
		}
		model = g
		schedule = "Grid mesh clock (4 ticks/call)"
	} else if mode.IsLineStep() {
		schedule = "Stack Line pipeline"
	}
	s, e := nativecuda.Mount(model)
	if e != nil {
		return e
	}
	defer s.Close()
	x := core.NewTensor[float32](2, 4)
	for i := range x.Data {
		x.Data[i] = float32(i%5-2) * .1
	}
	target := core.NewTensor[float32](2, 4)
	var loss float64
	for i := 0; i < steps; i++ {
		loss, e = s.TrainMSE(x, target, mode, .01, 4)
		if e != nil {
			return fmt.Errorf("%s: %w", mode, e)
		}
	}
	if e = s.SyncToGo(); e != nil {
		return e
	}
	fmt.Printf("mode=%s schedule=%s calls=%d loss=%.8g weights=synced-to-Go\n", mode, schedule, steps, loss)
	return nil
}

func cudaPlatformOK() bool {
	return runtime.GOARCH == "amd64" && (runtime.GOOS == "linux" || runtime.GOOS == "windows")
}