rapid flickering during loading a new scene

[coderofsalvation]

On Android and Oculus Quest2 (in immersive mode), it happens quite often that after clicking a portal, rapid flickering occurs (basically alternating between the current scene and a completely white image).
I've attached a (vesta) portal to reproduce it, but beware it never happens on desktop, but many times on oculus quest (and sometimes on android).

I can't tell whether its the JML causing it or not, perhaps one of these global vesta-scripts are used by various scenes, and are choking on something?

<FireBoxRoom>
 <Inputs>
  <Attribute description="URL for Image 1" name="image_1" type="text" value="https://picsum.photos/1920/1080/?random"/>
  <Attribute description="URL for Image 2" name="image_2" type="text" value="https://picsum.photos/1280/720/?random"/>
  <Attribute description="URL for Video" name="video_1" type="text" value="https://www.youtube.com/watch?v=jBNxhmtJLP8"/>
  <Attribute description="Autoplay Video?" name="video_autoplay" type="boolean" value="false"/>
  <Attribute description="URL for portal at entrance" name="link_url" type="text" value="https://vesta.janusxr.org"/>
  <Attribute description="Thumbnail for entrance portal" name="link_thumbnail" type="text" value="https://vesta.janusxr.org/files/admin/default_webspace/vesta_link.jpg"/>
 </Inputs>
 <Assets>
  
  <AssetObject id="main" src="https://vesta.janusxr.org/files/admin/default_webspace/TheMinimalistPalace.dae.gz"/>
  <AssetObject id="main_col" src="https://vesta.janusxr.org/files/admin/default_webspace/TheMinimalistPalaceCol.obj.gz"/>
  
  <AssetObject id="pic1" src="https://vesta.janusxr.org/files/admin/default_webspace/pic1.obj"/>
  <AssetObject id="pic2" src="https://vesta.janusxr.org/files/admin/default_webspace/pic2.obj"/>
  
  <AssetImage id="pic1_img" src="https://picsum.photos/1920/1080/?random"/>
  <AssetImage id="pic2_img" src="https://picsum.photos/1280/720/?random"/>
  <AssetImage id="link_img" src="https://vesta.janusxr.org/files/admin/default_webspace/vesta_link.jpg"/>
  
  <AssetObject id="youtube1" src="https://vesta.janusxr.org/files/admin/default_webspace/youtube1.obj"/>
  <AssetVideo auto_play="true" gain="0.3" id="main_video" src="https://www.youtube.com/watch?v=jBNxhmtJLP8"/>
  
  <AssetImage id="irradiance" src="https://vesta.janusxr.org/files/admin/default_webspace/Skybox/HDRcubemap_irradiance64.dds.gz" tex_clamp="true"/>
  <AssetImage id="radiance" src="https://vesta.janusxr.org/files/admin/default_webspace/Skybox/HDRcubemap_radiance256.dds.gz" tex_clamp="true"/>
  <AssetImage id="probe" src="https://vesta.janusxr.org/files/admin/default_webspace/Skybox/TheMinimalistPalaceWebProbe.jpg"/>
  
  <AssetObject id="carpet" src="https://vesta.janusxr.org/files/admin/default_webspace/TheMinimalistPalaceCarpet.obj.gz"/>
  <AssetScript src="https://vesta.janusxr.org/files/admin/default_webspace/script.js"/>
  <AssetScript src="https://vesta.janusxr.org/static/js/live_update.js"/>
  <AssetScript src="https://vesta.janusxr.org/files/admin/scripts/vesta_comment_inject.js"/>
 </Assets>
 <Room cubemap_irradiance_id="irradiance" cubemap_radiance_id="radiance" pbr="true" pos="-2.8 1 0" use_local_asset="room_plane" visible="false" xdir="0 0 -1" zdir="1 0 0">
  <Object collision_id="main_col" envmap_id="probe" id="main" js_id="Scan_id_forJS"/>
  <Object id="pic1" image_id="pic1_img" lighting="false"/>
  <Object id="pic2" image_id="pic2_img" lighting="false"/>
  <Object collision_id="youtube1" id="youtube1" lighting="false" video_id="main_video"/>
  <Link draw_text="false" pos="-2.95 0.8 0" scale="2 3.2 1" thumb_id="link_img" url="https://vesta.janusxr.org" zdir="1 0 0"/>
  
  <Object fwd="0.542248 0 -0.840219" js_id="vesta_thumbnail_camera" pos="5.2 2 0"/>
 </Room>
</FireBoxRoom>

https://vesta.janusxr.org/files/admin/default_webspace/script.js

room.onLoad = function() 
{
	var timetest = room.url.length % 10;
	room.loadNewAsset("Image", {id:"carpet", src:"https://vesta.janusvr.com/files/admin/default_webspace/carpets/carpet" + timetest + ".jpg"})
	room.createObject("Object", {id:"carpet", js_id:"carpet", image_id:"carpet"});
	
	if (room.objects['Scan_id_forJS'])
	{
		room.objects["Scan_id_forJS"].col = Vector(0.5+0.5*Math.random(), 0.5+0.5*Math.random(), 0.5+0.5*Math.random());
	}
}

https://vesta.janusxr.org/static/js/live_update.js

var live_update_started = false;
var userid;
var lighting_fix = false;
var loaded = false;

room.update = function(dt)
{
	if (!isJanusWeb)
	{
		updateTimers();
	}
	if (!loaded)
	{
		loaded = true;
		userid = player.userid;
		updateUserInfo();
	}
}
function get_server_base()
{
	var protocol = 'http';
	var hostname = extractHostname(room.url)
	
    if (room.url.indexOf("://") > -1) {
        protocol = room.url.split('/')[0];
    }
	
	return protocol+'//'+hostname;
}
function extractHostname(url) {
    var hostname;
    /* find & remove protocol (http, ftp, etc.) and get hostname */

    if (url.indexOf("://") > -1) {
        hostname = url.split('/')[2];
    }
    else {
        hostname = url.split('/')[0];
    }

    /* find & remove port number */
    /*  hostname = hostname.split(':')[0]; */
    /* find & remove "?" */
    hostname = hostname.split('?')[0];

    return hostname;
}

function updateUserInfo()
{
	var xhr = new XMLHttpRequest();
	xhr.addEventListener('load', updateUserInfoDone);
	var server_base = get_server_base();
	xhr.open('POST', server_base+'/user_count');
	if (isJanusWeb)
	{
		xhr.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
	}
	var data = 'started='+live_update_started+'&disableParty=false&username='+encodeURIComponent(userid)+'&roomUrl='+encodeURIComponent(player.url);
	live_update_started = true;
	xhr.send(data);
}

function updateUserInfoDone(ev)
{
	setTimeout(function()
	{
		updateUserInfo()
	}, 10000);
}


/* Native client compatibility functions */
var isJanusWeb = (typeof elation != 'undefined');

function getRandomIntInclusive(min, max) {
	min = Math.ceil(min);
	max = Math.floor(max);
	return Math.floor(Math.random() * (max - min + 1)) + min;
}
function getTimestamp() {
	if (isJanusWeb) {
		return performance.now();
	}
	return new Date().getTime();
}
 
if (typeof setTimeout != 'function') {
	var globalTimers = {},
			globalTimerID = 1;
 
	setTimeout = function(fn, delay) {
		var now = getTimestamp(),
				id = globalTimerID++;
		globalTimers[id] = {fn: fn, delay: delay, time: now, args: Array.prototype.splice.call(arguments, 2), repeat: false};
		return id;
	}
	setInterval = function(fn, delay) {
		var now = getTimestamp(),
				id = globalTimerID++;
		globalTimers[id] = {fn: fn, delay: delay, time: now, args: Array.prototype.splice.call(arguments, 2), repeat: true};
		return id;
	}
	clearTimeout = function(id) {
		delete globalTimers[id];
	}
	updateTimers = function() {
		var now = getTimestamp();
		var keys = Object.keys(globalTimers);
		for (var i = 0; i < keys.length; i++) {
			var id = keys[i],
					timer = globalTimers[id];
			if (timer.time + timer.delay <= now) {
				timer.fn.apply(null, timer.args);
				if (timer.repeat) {
					timer.time = now;
				} else {
					clearTimeout(id);
				}
			}
		}
	}
}
 
function bind(ctx, fn) {
	if (typeof fn == 'function') {
		var fnargs = Array.prototype.splice.call(arguments, 2);
		fnargs.unshift(ctx);
		return (typeof fn.bind == 'function' ?
				Function.prototype.bind.apply(fn, fnargs) :
				function() { fn.apply(ctx, arguments); }
			);
	} else if (typeof ctx == 'function') {
		return ctx;
	}
}

https://vesta.janusxr.org/files/admin/scripts/vesta_comment_inject.js

var isJanusWeb = (typeof elation != 'undefined');
var api_query_url = null;
var comments = [];
var comment_positions = [];
var fetched = false;
var comment_bubble_scale = 0.2;
var comment_scale = 0.5;
var spin_speed = 30;
var move_timer = 0;
var reload_timer = 0;
var resize_timer = 0;
var comments_enabled = true;


// native
var last_gaze_js_id = null;
var gaze_time = 0.0;
var gaze_on = false;

function get_server_base()
{
	var protocol = 'http';
	var hostname = extractHostname(room.url)
	
    if (room.url.indexOf("://") > -1) {
        protocol = room.url.split('/')[0];
    }
	
	return protocol+'//'+hostname;
}
function extractHostname(url) {
    var hostname;
    //find & remove protocol (http, ftp, etc.) and get hostname

    if (url.indexOf("://") > -1) {
        hostname = url.split('/')[2];
    }
    else {
        hostname = url.split('/')[0];
    }

    //find & remove port number
    // hostname = hostname.split(':')[0];
    //find & remove "?"
    hostname = hostname.split('?')[0];

    return hostname;
}
function getComments()
{
	var xhr = new XMLHttpRequest();
	xhr.addEventListener('load', parseComments);
	xhr.open('GET', api_query_url);
	xhr.send();
}
function parseComments(ev)
{
	var xhr = ev.target;
	var response = JSON.parse(xhr.responseText);

	if (response.success && response.data)
	{
		for (var i = 0; i < response.data.length; i++)
		{
			var comment = response.data[i];
			var pos = comment.pos
			if (pos)
			{
				var cid = comment.id;
				var js_id = 'vestacommentinject_'+cid;
				if (room.objects[js_id] == null)
				{
					var startpos = [0,0,0];
					if (room.playerstartposition)
					{
						startpos = room.playerstartposition;
					}
					var dist = distance(Vector(pos.x, pos.y+1.0, pos.z), Vector(startpos[0], startpos[1], startpos[2]))
					var valid = true;
					if (dist > 1.5)
					{
						for (i2 = 0; i2 < comment_positions.length; i2 ++)
						{
							var pos2 = comment_positions[i2];
							dist = distance(Vector(pos.x, pos.y+1.0, pos.z), pos2);
							if (dist <= comment_bubble_scale*2 && valid)
							{
								//console.log('Skip comment '+cid+', too close to another');
								valid = false;
							}
							
						}
						if (valid)
						{
							var image = comment.image;
							var username = comment.username;
							var css = ".textwrapper { padding: 16px 16px 16px 16px; } .wrapper { font-size: 32px; color: white; border-style:solid; border-width:4px; border-color: #343138; background-color: #1d1c1f; color: white; font-family: sans-serif; } h1 { font-size: 38px; display:table-cell; padding-top: 8px} p { font-size: 32px; }";
							var body = '<div class="wrapper"> <div class="textwrapper"> <h1>'+username+'</h1> <p>'+comment.body+'</p> </div> </div>';
							var bubble;
							if (isJanusWeb)
							{
								bubble = room.createObject('Object', { id:'sphere', collision_radius:comment_bubble_scale, collision_trigger:true, js_id:'bubble_'+js_id, image_id: get_server_base()+'/static/images/comment_equi.jpg', pos:Vector(pos.x, pos.y+1.0, pos.z), gazetime:'200', ongazeactivate:'show_comment("'+js_id+'")', ongazeleave:'hide_comment("'+js_id+'")', ongazeprogress:"gazeProgress(this, event)", scale:Vector(comment_bubble_scale,comment_bubble_scale,comment_bubble_scale), locked:true, lighting:false, cull_face:'front', rotate_deg_per_sec:spin_speed });
							}
							else
							{
								bubble = room.createObject('Object', { id:'sphere', collision_id:'sphere', collision_scale:Vector(1), collision_trigger:true, js_id:'bubble_'+js_id, image_id: get_server_base()+'/static/images/comment_equi.jpg', pos:Vector(pos.x, pos.y+1.0, pos.z), gazetime:'200', ongazeactivate:'show_comment("'+js_id+'")', ongazeleave:'hide_comment("'+js_id+'")', ongazeprogress:"gazeProgress(this, event)", scale:Vector(comment_bubble_scale,comment_bubble_scale,comment_bubble_scale), locked:true, lighting:false, cull_face:'front', rotate_deg_per_sec:spin_speed });
							}
							var bubble_outside = room.createObject('Object', { id:'sphere', js_id:'bubble_outside_'+js_id, col:Vector(1,1,1), pos:Vector(pos.x, pos.y+1.0, pos.z), scale:Vector(comment_bubble_scale,comment_bubble_scale,comment_bubble_scale), locked:true, lighting:false, cull_face:'front', rotate_deg_per_sec:spin_speed });
							var comment_obj = null;
							if (image) // use rendered image
							{
								room.loadNewAsset('Image', {id:image, src:image});
								var comment_obj = room.createObject('Image', { js_id:js_id, collision_trigger:true, id:image, css: css, pos:Vector(pos.x, pos.y+1, pos.z), font_size:32, text_col: Vector(1,1,1), back_alpha: 0, back_col: Vector(0.114, 0.110, 0.122), locked:true, lighting: false, visible:false, cull_face:'none', scale:Vector(comment_scale,comment_scale,comment_scale*0.001) });
							}
							else // fallback paragraph. doesn't work in firefox
							{
								var comment_obj = room.createObject('Paragraph', { js_id:js_id, collision_trigger:true, text:body, css: css, pos:Vector(pos.x, pos.y+1, pos.z), font_size:32, text_col: Vector(1,1,1), back_alpha: 0, back_col: Vector(0.114, 0.110, 0.122), locked:true, lighting: false, visible:false, cull_face:'none', scale:Vector(comment_scale,comment_scale,comment_scale) });
							}
							comments.push(js_id);
							comment_positions.push(Vector(pos.x, pos.y+1.0, pos.z));
							//console.log('Injected comment '+cid);
						}
					}
					else
					{
						//console.log('Skip comment '+cid+', too close to start');
					}
				}
			}
		}
		if (comments.length > 15)
		{
			for (var i = 0; i < comments.length-15; i++)
			{
				var js_id = comments[i];
				var js_id1 = 'bubble_'+comments[i];
				var js_id2 = 'bubble_outside_'+comments[i];
				room.removeObject(js_id);
				room.removeObject(js_id1);
				room.removeObject(js_id2);
				//console.log('Removed extra comment '+js_id);
			}
		}
	}
}
function show_comment(js_id)
{
	var bubble = room.objects['bubble_'+js_id];
	var obj = room.objects[js_id];
	if (bubble && obj && comments_enabled)
	{
		bubble.visible = false;
		if (isJanusWeb)
		{			
			bubble.collision_radius = comment_bubble_scale*2;
		}
		else
		{
			bubble.collision_scale = Vector(2);
		}
		obj.visible = true;
	}
}
function hide_comment(js_id)
{
	var bubble = room.objects['bubble_'+js_id];
	var obj = room.objects[js_id];
	if (bubble && obj && comments_enabled)
	{
		bubble.visible = true;
		obj.visible = false;
		var rescale = true;
		if (isJanusWeb)
		{
			if (bubble.colliders.children.length > 0)
			{
				rescale = false;
			}
		}
		if (rescale)
		{
			if (isJanusWeb)
			{
				bubble.collision_radius = comment_bubble_scale;
			}
			else
			{
				bubble.collision_scale = Vector(1);
			}
		}
		bubble.rotate_deg_per_sec = spin_speed;
	}
}
function gazeProgress(object, ev)
{
	var speed = spin_speed - (spin_speed * ev.data);
	object.rotate_deg_per_sec = speed;
}
room.onload = function()
{
	if (!isJanusWeb)
	{
		room.comments_enabled = true;
	}
}
room.enable_comments = function()
{
	for (i in comments)
	{
		var js_id1 = 'bubble_'+comments[i];
		var bubble_out = room.objects['bubble_outside_'+comments[i]];
		var obj = room.objects[js_id1];
		var obj2 = room.objects[comments[i]];
		if (obj && obj2 && bubble_out)
		{
			obj.visible = true;
			obj2.visible = false;
			bubble_out.visible = true;
		}
	}
}
room.update = function(dt)
{
	if (isJanusWeb)
	{
		if (room.comments_enabled !== 'undefined')
		{
			comments_enabled = room.comments_enabled;
		}
	}
	else
	{
		comments_enabled = true;
	}
	if (api_query_url == null)
	{
		api_query_url = get_server_base()+'/api/get_comments?url='+room.url+'&limit=15';
	}
	if (!isJanusWeb)
	{
		var cursor_obj = player.cursor_object;
		if (last_gaze_js_id == 'bubble_'+cursor_obj)
		{
			cursor_obj = 'bubble_'+cursor_obj
		}
		if (cursor_obj != last_gaze_js_id)
		{
			gaze_time = 0;
		}
		var dist = distance(room.objects[player.cursor_object], player.pos);
		if (dist < 0.3)
		{
			gaze_time = 0;
		}
		if (gaze_time >= 200 && cursor_obj.length > 7)
		{
			if (!gaze_on)
			{
				if (room.objects[cursor_obj.substr(7)])
				{
					show_comment(cursor_obj.substr(7));
					gaze_on = true;
				}
			}
		}
		else
		{
			if (gaze_on)
			{
				hide_comment(last_gaze_js_id.substr(7));
				gaze_on = false;
			}
			else
			{
				gaze_time += dt;
			}
		}
		last_gaze_js_id = cursor_obj;
	}
	if (isJanusWeb || true)
	{
		reload_timer -= dt*0.001;
		move_timer -= dt*0.001;
		resize_timer -= dt*0.001;
		
		if (reload_timer <= 0)
		{
			fetched = false;
			reload_timer += 60;
		}
		
		if (comments_enabled)
		{
			if (fetched == false)
			{
				fetched = true;
				getComments();		
			}
			for (i in comments)
			{
				var js_id1 = 'bubble_'+comments[i];
				var bubble_out = room.objects['bubble_outside_'+comments[i]];
				var obj = room.objects[js_id1];
				var obj2 = room.objects[comments[i]];
				if (obj && obj2)
				{
					//var dist = distance(obj.pos, player.pos);
					if (resize_timer <= 0)
					{
						var dist2 = distance(obj.pos, player.head_pos);
						var scale = comment_bubble_scale+dist2*0.01;
						bubble_out.scale = Vector(scale,scale,scale);
					}
					if (true)
					{
						if (obj2.visible)
						{
							//var angle_from = Math.atan2(player.pos.z - obj.pos.z, player.pos.x - obj.pos.x);
							//var fwd = Vector(Math.cos(angle_from),0,Math.sin(angle_from));
							obj2.ydir = Vector(0,1,0);
							obj2.zdir = scalarMultiply(player.view_dir, -1);
							obj2.xdir = cross(obj2.ydir, obj2.zdir);
							obj2.fwd = scalarMultiply(player.view_dir, -1);
							obj2.pos = Vector(obj.pos); //translate(obj.pos, scalarMultiply(obj2.fwd, -1));
							if (isJanusWeb)
							{
								obj2.updateOrientationFromDirvecs();
							}
						}
						else
						{
							if (move_timer <= 0)
							{
								obj2.pos.y = player.pos.y - 200;
								//obj2.visible = false;
							}
						}
					}
					bubble_out.visible = obj.visible;
				}
			}
		}
		else
		{
			for (i in comments)
			{
				var js_id1 = 'bubble_'+comments[i];
				var bubble_out = room.objects['bubble_outside_'+comments[i]];
				var obj = room.objects[js_id1];
				var obj2 = room.objects[comments[i]];
				if (obj && obj2 && bubble_out)
				{
					obj.visible = false;
					obj2.visible = false;
					bubble_out.visible = false;
				}
			}
		}
		if (resize_timer <= 0)
		{
			resize_timer = 0.1;
		}
		if (move_timer <= 0)
		{
			move_timer = 2;
		}
	}
}

[jbaicoianu]

Most of this happens because we don't pause the renderer while loading new rooms. One of the strengths of Janus is the fact that it can load new scenes on the fly, but this always leads to dropped frames as the GPU pauses to receive new textures, etc - and since we load progressively, these little hiccups can happen over a period of a few seconds, depending on the room being loaded. On desktop and mobile this isn't a problem, you might feel a few hiccups in movement but the overall experience is fine - in VR though, especially standalone-class devices, the renderer gets bogged down and ASW starts bugging out if we drop too many frames in a row.

There's definitely some optimization to be had - we participated in a few attempts over the years to improve Three.js' ability to stream textures to the GPU with minimal blocking, but I think we're not using the latest and greatest there. Upgrading Three.js to the latest version and making use of the more modern loaders could help.

There are also some memory-related optimizations we could do which might also help, there's a non-zero amount of preprocessing we do on the images, eg, detecting transparency so we can automatically set the material properties, and these all lead to microstutters during loading.

I think I experimented with disabling the renderer while a portal transition was happening while in VR, but I'll have to poke around to see if that's enabled or not. This did improve the loading jankiness, but it meant staring at a black screen for often longer than seemed reasonable.

In my mind I've got a whole new architecture which uses OffscreenCanvas and workers to make sure that we can always render frames even if a room is loading, but that would definitely be something for v2.

[coderofsalvation]

Maybe we should pause this issue for a bit (perhaps add a v2 tag on this issue?).

I mean, otherwise we might risk working 2 times on this issue in different ways (for v1 and v2).
Sometimes old bugs magically dissappear when refactoring (and introduce new issues elsewhere) 😸

Perhaps the takeway is: when the time is right, lets test v2 in VR with heavy rooms which link heavy rooms (large textures) to catch shenanigans fast.

Also, I think I can avoid the situation by keeping my textures & geometry small, and avoid linking to heavier rooms.

[coderofsalvation]

update: I think I've discovered something...it might not be related to rendering..but to handtracking!

reason: it only happens during clicking a portal..and...I've noticed that after I activated a portal via the laser (pinch) I see the hand-model rapidly pinch over-and-over again..as if it's stuck in a loop.

I'm going to log select/click-events to see what is up.
Perhaps we can resolve this by temporarily disabling click-events when a new portal is loaded.