Skip to content

Kitchen Sink

The "Kitchen Sink" reality tabs example is a showcase of a few of the Magic Leap APIs: meshing, eye-tracking, plane-tracking, hand-tracking, just about everything but the kitchen sink!

Magic Leap API

The following are how the example utilizes Exokit's Magic Leap API.

  • Enabling

          let enabled = false;
          const _enable = () => {
            mesher = window.browser.magicleap.RequestMeshing();
            mesher.onmesh = _onMesh;
            planeTracker = window.browser.magicleap.RequestPlaneTracking();
            planeTracker.onplanes = _onPlanes;
            handTracker = window.browser.magicleap.RequestHandTracking();
            handTracker.onhands = _onHands;
            handMesh.visible = true;
            eyeTracker = window.browser.magicleap.RequestEyeTracking();
            enabled = true;
          };
    

  • Meshing / _onMesh

          const _onMesh = updates => {
            for (let i = 0; i < updates.length; i++) {
              const update = updates[i];
              const {id, type} = update;
              if (type === 'new' || type === 'update') {
                _loadTerrainMesh(_getTerrainMesh(id), update);
              } else if (type === 'unchanged') {
                // nothing
              } else {
                const index = terrainMeshes.findIndex(terrainMesh => terrainMesh.meshId === id);
                if (index !== -1) {
                  const terrainMesh = terrainMeshes[index];
                  _removeTerrainMesh(terrainMesh);
                  terrainMeshes.splice(index, 1);
                }
              }
            }
          };
    

  • Eye tracking / eyeTracker

          if (eyeTracker) {
            const {fixation} = eyeTracker;
            const {position, rotation} = fixation;
            eyeMesh.position.fromArray(position);
            eyeMesh.quaternion.fromArray(rotation);
            eyeMesh.updateMatrix();
            eyeMesh.updateMatrixWorld();
            eyeMesh.visible = true;
          } else {
            eyeMesh.visible = false;
          }
    

  • Plane-tracking / planeMeshes / _onPlanes

          const planeMeshes = [];
          const planeGeometry = new THREE.PlaneBufferGeometry(1, 1);
          const planeMaterial = new THREE.MeshPhongMaterial({
            color: 0xFF0000,
          });
    
          const _loadPlanes = planes => {
            _clearPlanes();
            for (let i = 0; i < planes.length; i++) {
              const plane = planes[i];
              const mesh = new THREE.Mesh(planeGeometry, planeMaterial);
              mesh.frustumCulled = false;
              mesh.position.fromArray(plane.position);
              mesh.quaternion.fromArray(plane.rotation);
              mesh.scale.set(plane.size[0], plane.size[1], 1);
              mesh.updateMatrix();
              mesh.updateMatrixWorld();
              scene.add(mesh);
              planeMeshes.push(mesh);
            }
          };
    
          const _clearPlanes = () => {
            for (let i = 0; i < planeMeshes.length; i++) {
              scene.remove(planeMeshes[i]);
            }
            planeMeshes.length = 0;
          };
    
          const _onPlanes = planes => {
            _loadPlanes(planes);
          };
    

  • Hand-tracking / handMesh / _onHands

            const mesh = new THREE.Mesh(geometry, material);
            mesh.update = hands => {
              let positionIndex = 0;
              let indexIndex = 0;
              const _shiftIndex = array => {
                for (let l = 0; l < array.length; l++) {
                  array[l] += positionIndex / 3;
                }
              };
              for (let i = 0; i < hands.length; i++) {
                const hand = hands[i];
                const {pointer, grip, wrist, fingers} = hand;
                const allFingers = [wrist].concat(fingers);
                for (let j = 0; j < allFingers.length; j++) {
                  const bones = allFingers[j];
                  for (let k = 0; k < bones.length; k++) {
                    const positionFloat32Array = bones[k];
                    if (positionFloat32Array) {
                      const position = localVector.fromArray(positionFloat32Array);
                      const newGeometry = boneGeometry.clone()
                        .applyMatrix(
                          localMatrix.makeTranslation(position.x, position.y, position.z)
                        );
                      _shiftIndex(newGeometry.index.array);
                      positions.set(newGeometry.attributes.position.array, positionIndex);
                      positionIndex += newGeometry.attributes.position.array.length;
                      indices.set(newGeometry.index.array, indexIndex);
                      indexIndex += newGeometry.index.array.length;
                    }
                  }
                }
    
                if (pointer) {
                  const newGeometry = pointerGeometry.clone()
                    .applyMatrix(
                      localMatrix.compose(
                        localVector.fromArray(pointer.position),
                        localQuaternion.fromArray(pointer.rotation),
                        localVector2.set(1, 1, 1),
                      )
                    );
                  _shiftIndex(newGeometry.index.array);
                  positions.set(newGeometry.attributes.position.array, positionIndex);
                  positionIndex += newGeometry.attributes.position.array.length;
                  indices.set(newGeometry.index.array, indexIndex);
                  indexIndex += newGeometry.index.array.length;
                }
    
                if (grip) {
                  const newGeometry = pointerGeometry.clone()
                    .applyMatrix(
                      localMatrix.compose(
                        localVector.fromArray(grip.position),
                        localQuaternion.fromArray(grip.rotation),
                        localVector2.set(1, 1, 1),
                      )
                    );
                  _shiftIndex(newGeometry.index.array);
                  positions.set(newGeometry.attributes.position.array, positionIndex);
                  positionIndex += newGeometry.attributes.position.array.length;
                  indices.set(newGeometry.index.array, indexIndex);
                  indexIndex += newGeometry.index.array.length;
                }
              }
    
              positionAttribute.needsUpdate = true;
              indexAttribute.needsUpdate = true;
              geometry.setDrawRange(0, indexIndex);
            };
            mesh.visible = false;
            mesh.frustumCulled = false;
            return mesh;
          })();
          scene.add(handMesh);
    
          const _onHands = hands => {
            handMesh.update(hands);
          };