encoder.js
function getFloatScale( values, min, max, error ) {
	var mid = (min + max) / 2, range = mid - min,
		norm_8 = (range / INT8_MAX),
		norm_16 = (range / INT16_MAX),
		ok_8 = true, ok_16 = true,
		er_8 = 0, er_16 = 0,
		v, uv, er;

	// For the values given, check if 8-bit or 16-bit
	// scaling brings smaller error value
	for (var i=0, l=values.length; i<l; ++i) {

		// Test 8-bit scaling
		if (ok_8) {
			v = Math.round((values[i] - mid) / norm_8);
			uv = v * norm_8 + mid;
			er = (uv-v)/v;
			if (er > er_8) er_8 = er;
			if (er >= error) {
				ok_8 = false;
			}
		}

		// Test 16-bit scaling
		if (ok_16) {
			v = Math.round((values[i] - mid) / norm_16);
			uv = v * norm_16 + mid;
			er = (uv-v)/v;
			if (er > er_16) er_16 = er;
			if (er >= error) {
				ok_16 = false;
			}
		}

		if (!ok_8 && !ok_16)
			return [ 0, NUMTYPE.UNKNOWN ];
	}

	// Pick most appropriate normalization factor
	if (ok_8 && ok_16) {
		if (er_8 < er_16) {
			return [ norm_8, NUMTYPE.INT8 ];
		} else {
			return [ norm_16, NUMTYPE.INT16 ];
		}
	} else if (ok_8) {
		return [ norm_8, NUMTYPE.INT8 ];
	} else if (ok_16) {
		return [ norm_16, NUMTYPE.INT16 ];
	} else {
		return [ 0, NUMTYPE.UNKNOWN ];
	}

}
function getFloatScale( values, min, max, error ) {
	var mid = (min + max) / 2, range = mid - min,
		norm_8 = (range / INT8_MAX),
		norm_16 = (range / INT16_MAX),
		ok_8 = true, ok_16 = true,
		er_8 = 0, er_16 = 0,
		v, uv, er;

	// For the values given, check if 8-bit or 16-bit
	// scaling brings smaller error value
	for (var i=0, l=values.length; i<l; ++i) {

		// Test 8-bit scaling
		if (ok_8) {
			v = Math.round((values[i] - mid) / norm_8);
			uv = v * norm_8 + mid;
			er = (uv-v)/v;
			if (er > er_8) er_8 = er;
			if (er >= error) {
				ok_8 = false;
			}
		}

		// Test 16-bit scaling
		if (ok_16) {
			v = Math.round((values[i] - mid) / norm_16);
			uv = v * norm_16 + mid;
			er = (uv-v)/v;
			if (er > er_16) er_16 = er;
			if (er >= error) {
				ok_16 = false;
			}
		}

		if (!ok_8 && !ok_16)
			return [ 0, NUMTYPE.UNKNOWN ];
	}

	// Pick most appropriate normalization factor
	if (ok_8 && ok_16) {
		if (er_8 < er_16) {
			return [ norm_8, NUMTYPE.INT8 ];
		} else {
			return [ norm_16, NUMTYPE.INT16 ];
		}
	} else if (ok_8) {
		return [ norm_8, NUMTYPE.INT8 ];
	} else if (ok_16) {
		return [ norm_16, NUMTYPE.INT16 ];
	} else {
		return [ 0, NUMTYPE.UNKNOWN ];
	}

}
function getFloatScale ( values , min , max , error ) { var mid = ( min + max ) / 2 , range = mid - min , norm_8 = ( range / INT8_MAX ) , norm_16 = ( range / INT16_MAX ) , ok_8 = true , ok_16 = true , er_8 = 0 , er_16 = 0 , v , uv , er ; // For the values given, check if 8-bit or 16-bit // scaling brings smaller error value for ( var i = 0 , l = values . length ; i < l ; ++ i ) { // Test 8-bit scaling if ( ok_8 ) { v = Math . round ( ( values [ i ] - mid ) / norm_8 ) ; uv = v * norm_8 + mid ; er = ( uv - v ) / v ; if ( er > er_8 ) er_8 = er ; if ( er >= error ) { ok_8 = false ; } } // Test 16-bit scaling if ( ok_16 ) { v = Math . round ( ( values [ i ] - mid ) / norm_16 ) ; uv = v * norm_16 + mid ; er = ( uv - v ) / v ; if ( er > er_16 ) er_16 = er ; if ( er >= error ) { ok_16 = false ; } } if ( ! ok_8 && ! ok_16 ) return [ 0 , NUMTYPE . UNKNOWN ] ; } // Pick most appropriate normalization factor if ( ok_8 && ok_16 ) { if ( er_8 < er_16 ) { return [ norm_8 , NUMTYPE . INT8 ] ; } else { return [ norm_16 , NUMTYPE . INT16 ] ; } } else if ( ok_8 ) { return [ norm_8 , NUMTYPE . INT8 ] ; } else if ( ok_16 ) { return [ norm_16 , NUMTYPE . INT16 ] ; } else { return [ 0 , NUMTYPE . UNKNOWN ] ; } }
Calculate and return the numerical type and the scale to
apply to the float values given in order to minimize the error.
Calculate and return the numerical type and the scale to apply to the float values given in order to minimize the error .