Audio, MIDI, Sensors, and AI #

// Start the audio context explicitly.
sys.audio.start();
// Set a comfortable master volume.
sys.audio.setMasterGain(0.5);

// Create an oscillator and keep its runtime handle.
const osc = sys.audio.createOscillator();
// Choose a simple sine wave.
sys.audio.setOscillatorType(osc, 'sine');
// Tune the oscillator to A3.
sys.audio.setOscillatorFrequency(osc, 220);
// Set the oscillator gain before playback begins.
sys.audio.setOscillatorGain(osc, 0.25);
// Start continuous playback for this oscillator.
sys.audio.startOscillator(osc);

// Configure a short attack and release envelope.
sys.audio.setOscillatorEnvelope(osc, 0.01, 0.12, 0.7, 0.25);
// Trigger the note on the configured oscillator.
sys.audio.noteOn(osc);
// Release the note after 300 milliseconds.
sys.timer.once(300, () => sys.audio.noteOff(osc));

// Choose the sample rate for the generated buffer.
const sampleRate = 44100;
// Allocate one second of mono samples.
const samples = new Array(sampleRate);

// Fill the buffer with a quiet 110 Hz sine wave.
for (let i = 0; i < samples.length; i++) {
  // Convert the sample index into a phase angle.
  samples[i] = Math.sin((i / sampleRate) * Math.PI * 2 * 110) * 0.25;
}

// Create a runtime audio buffer for the samples.
const buffer = sys.audio.createBuffer(sampleRate, 1, samples.length);
// Upload the generated samples starting at offset 0.
sys.audio.setBufferData(buffer, samples, 0);
// Play the buffer once at full gain.
const playback = sys.audio.playBuffer(buffer, false, 1.0);

// Decode a project asset into a runtime audio buffer.
const hit = sys.audio.loadBuffer('sounds/hit.ogg');
const hitFromNetwork = sys.audio.loadBufferFromBuffer(await (await fetch(url)).arrayBuffer());

// Check for a load failure before using the returned handle.
if (hit < 0) {
  // Print the most recent audio error to help diagnose missing or invalid assets.
  console.log(sys.audio.getError());
}

// Play the decoded sound once at a comfortable level.
const hitPlayback = sys.audio.playBuffer(hit, false, 0.8);

// Load a background loop from the bundled project assets.
const ambience = sys.audio.loadBuffer('audio/ambience.mp3');

// Start the buffer in looping mode at a low gain.
const ambiencePlayback = sys.audio.playBuffer(ambience, true, 0.35);

// Stop the looping playback later when the scene changes.
sys.audio.stopBuffer(ambiencePlayback);

// Refresh devices before showing a picker.
sys.midi.refreshDevices();

// Iterate through the available MIDI inputs.
for (const device of sys.midi.getInputDevices()) {
  // Log the stable device id and human-readable name.
  console.log(device.id + ': ' + device.name);
}

// Open the first MIDI input and receive messages through a callback.
const input = sys.midi.openInput(0, (message) => {
  // Treat NOTE_ON with velocity greater than zero as a played note.
  if (message.type === sys.midi.NOTE_ON && message.data2 > 0) {
    // Log note number and velocity.
    console.log('Note ' + message.data1 + ' velocity ' + message.data2);
  }
});

// Open the first MIDI output.
const output = sys.midi.openOutput(0);
// Send middle C on channel 0.
sys.midi.noteOn(output, 0, 60, 100);
// Release middle C on channel 0.
sys.midi.noteOff(output, 0, 60, 0);
// Send channel pressure aftertouch on channel 0.
sys.midi.channelPressure(output, 0, 72);
// Send polyphonic pressure for note 60 on channel 0.
sys.midi.polyPressure(output, 0, 60, 64);
// Send a raw MIDI System Exclusive message.
sys.midi.sendRaw(output, [0xf0, 0x7e, 0x7f, 0x06, 0x01, 0xf7]);

// Create an RTP-MIDI session on an automatically selected port.
const session = sys.midi.createSession('Jam', 0);
// Receive MIDI messages from the network session.
sys.midi.onSessionMessage(session, (message) => {
  // Log the raw status byte for quick diagnostics.
  console.log('Network MIDI: ' + message.status);
});

// Connect to another RTP-MIDI peer.
sys.midi.connectSession(session, '192.168.1.20', 5004);
// Send a note through the session.
sys.midi.sessionNoteOn(session, 0, 64, 100);

// Start sensors only when the platform reports support.
if (sys.magneto.isAvailable()) {
  // Activate accelerometer and compass polling.
  sys.magneto.start();
}

// Read sensor state once per frame.
function frame() {
  // Get the latest accelerometer sample, if any.
  const accel = sys.magneto.getAccel();
  // Update tilt state only when a sample exists.
  if (accel) {
    // Store horizontal tilt.
    state.tiltX = accel.x;
    // Store vertical tilt.
    state.tiltY = accel.y;
  }

  // Get the latest compass sample, if any.
  const compass = sys.magneto.getCompass();
  // Update heading only when a compass sample exists.
  if (compass) {
    // Store heading in degrees.
    state.heading = compass.heading;
  }

  // Render the current sensor-driven state.
  draw();
  // Continue the frame loop.
  sys.animation.requestFrame(frame);
}

// Keep the display awake while the active experience needs it.
sys.device.keepScreenOn(true);

{
  "neural": true
}

// Check the capability before attempting to load ONNX models.
if (!sys.capabilities.neural.available) {
  // Choose a fallback path for builds without neural inference.
  console.log('No neural runtime in this build');
}

// Load an ONNX model from the project directory.
const model = sys.neural.loadModel('models/classifier.onnx');
const fetchedModel = sys.neural.loadModelFromBuffer(modelBytes);
// Inspect model inputs and outputs.
const info = sys.neural.getModelInfo(model);

// Use the first input name declared by the model.
const inputName = info.inputs[0].name;
// Use the first output name declared by the model.
const outputName = info.outputs[0].name;

// Allocate input data for a 1x3x224x224 tensor.
const data = new Float32Array(1 * 3 * 224 * 224);
// Run inference with a named input tensor.
const result = sys.neural.run(model, {
  // Bind the typed array to the model's input name.
  [inputName]: data
});

// Read the model output by name.
const output = result[outputName];

// Stage input data on the model handle.
sys.neural.setInput(model, inputName, data);
// Run inference using the staged inputs.
sys.neural.run(model);
// Read the staged output tensor by name.
const output = sys.neural.getOutput(model, outputName);

// Release model resources when they are no longer needed.
sys.neural.unloadModel(model);