Optimize Analyze Endpoint (#111)
* use unzip callback instead of unzipsync * use indents * move sortobj out of function * don't use res in the function * move some processing out of the color foreach * remove some object iterations * split header parse into a new function * switch statements * useless data assignment * "flatten" the object id lists * remove accidental timing thingies * use if in * group objects in the for loop * return of the in * remove requirement on app * add the try catch * use for loop instead of foreach * don't parse the header with the objs * use string * parse object key names along with object parsing * don't store is valid variable * don't use splice, it breaks the next request * create response.colors before parse
This commit is contained in:
zmx
GitHub
parent
5bfedbd24f
commit
8b37d43b45
1 changed files with 234 additions and 156 deletions
390
api/analyze.js
390
api/analyze.js
|
|
@ -6,73 +6,143 @@ const ids = require('../misc/objects.json')
|
|||
const blocks = require('../misc/blocks.json')
|
||||
|
||||
module.exports = async (app, req, res, level) => {
|
||||
let unencrypted = level.data.startsWith('kS') // some gdps'es don't encrypt level data
|
||||
let levelString = unencrypted ? level.data : Buffer.from(level.data, 'base64')
|
||||
|
||||
let unencrypted = level.data.startsWith('kS') // some gdps'es don't encrypt level data
|
||||
let levelString = unencrypted ? level.data : Buffer.from(level.data, 'base64')
|
||||
let response = {};
|
||||
let rawData;
|
||||
if (unencrypted) {
|
||||
const raw_data = level.data;
|
||||
|
||||
if (unencrypted) rawData = level.data
|
||||
zlib.unzip(levelString, (err, buffer) => {
|
||||
if (err) return res.send("-1"); // dumb
|
||||
rawData = buffer.toString('utf8');
|
||||
let data = rawData; // data is tweaked around a lot, so rawData is preserved
|
||||
const response_data = analyze_level(level, raw_data);
|
||||
return res.send(response_data);
|
||||
} else {
|
||||
zlib.unzip(levelString, (err, buffer) => {
|
||||
if (err) { return res.send("-1"); }
|
||||
|
||||
const raw_data = buffer.toString();
|
||||
const response_data = analyze_level(level, raw_data);
|
||||
return res.send(response_data);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function sortObj(obj, sortBy) {
|
||||
var sorted = {}
|
||||
var keys = !sortBy ? Object.keys(obj).sort((a,b) => obj[b] - obj[a]) : Object.keys(obj).sort((a,b) => obj[b][sortBy] - obj[a][sortBy])
|
||||
keys.forEach(x => {sorted[x] = obj[x]})
|
||||
return sorted
|
||||
}
|
||||
|
||||
function parse_obj(obj, splitter, name_arr, valid_only) {
|
||||
const s_obj = obj.split(splitter);
|
||||
let robtop_obj = {};
|
||||
|
||||
// semi-useless optimization depending on where at node js you're at
|
||||
for (let i = 0, obj_l = s_obj.length; i < obj_l; i += 2) {
|
||||
let k_name = s_obj[i];
|
||||
if (s_obj[i] in name_arr) {
|
||||
if (!valid_only) {
|
||||
k_name = name_arr[s_obj[i]];
|
||||
}
|
||||
robtop_obj[k_name] = s_obj[i + 1];
|
||||
}
|
||||
}
|
||||
return robtop_obj;
|
||||
}
|
||||
|
||||
function analyze_level(level, rawData) {
|
||||
let response = {};
|
||||
|
||||
let blockNames = Object.keys(blocks)
|
||||
let miscNames = Object.keys(ids.misc)
|
||||
let blockCounts = {}
|
||||
let miscCounts = {}
|
||||
let triggerGroups = []
|
||||
let highDetail = 0
|
||||
|
||||
data = data.split(";")
|
||||
let misc_objects = {};
|
||||
let block_ids = {};
|
||||
|
||||
function sortObj(obj, sortBy) {
|
||||
var sorted = {}
|
||||
var keys = !sortBy ? Object.keys(obj).sort((a,b) => obj[b] - obj[a]) : Object.keys(obj).sort((a,b) => obj[b][sortBy] - obj[a][sortBy])
|
||||
keys.forEach(x => {sorted[x] = obj[x]})
|
||||
return sorted
|
||||
for (const [name, object_ids] of Object.entries(ids.misc)) {
|
||||
const copied_ids = object_ids.slice(1);
|
||||
// funny enough, shift effects the original id list
|
||||
copied_ids.forEach((object_id) => {
|
||||
misc_objects[object_id] = name;
|
||||
});
|
||||
}
|
||||
|
||||
data.forEach((x, y) => {
|
||||
obj = app.parseResponse(x, ",")
|
||||
for (const [name, object_ids] of Object.entries(blocks)) {
|
||||
object_ids.forEach((object_id) => {
|
||||
block_ids[object_id] = name;
|
||||
});
|
||||
}
|
||||
|
||||
let keys = Object.keys(obj)
|
||||
keys.forEach((k, i) => {
|
||||
if (init.values[k]) k = obj[k][0]
|
||||
else if (properties[k]) obj[properties[k]] = obj[k]
|
||||
delete obj[k]
|
||||
})
|
||||
const data = rawData.split(";");
|
||||
const header = data.shift();
|
||||
|
||||
let id = obj.id
|
||||
if (ids.portals[id]) obj.portal = ids.portals[id]
|
||||
if (ids.orbs[id]) obj.orb = ids.orbs[id]
|
||||
if (ids.triggers[id]) obj.trigger = ids.triggers[id]
|
||||
let level_portals = [];
|
||||
let level_text = [];
|
||||
|
||||
if (obj.triggerGroups) obj.triggerGroups.split('.').forEach(x => triggerGroups.push(x))
|
||||
if (obj.highDetail == 1) highDetail += 1
|
||||
|
||||
blockNames.forEach(b => {
|
||||
if (blocks[b].includes(id)) {
|
||||
if (!blockCounts[b]) blockCounts[b] = 1
|
||||
else blockCounts[b] += 1
|
||||
}
|
||||
})
|
||||
|
||||
miscNames.forEach(b => {
|
||||
if (ids.misc[b].includes(Number(id))) {
|
||||
if (!miscCounts[b]) miscCounts[b] = [1, ids.misc[b][0]]
|
||||
else miscCounts[b][0] += 1
|
||||
}
|
||||
})
|
||||
|
||||
data[y] = obj;
|
||||
})
|
||||
let orb_array = {};
|
||||
let trigger_array = {};
|
||||
|
||||
let last = 0;
|
||||
let xArr = data.map(x => Number(x.x))
|
||||
let dl = data.length
|
||||
while (dl--) {last = xArr[dl] > last ? xArr[dl] : last}
|
||||
|
||||
const obj_length = data.length;
|
||||
for (let i = 0; i < obj_length; ++i) {
|
||||
obj = parse_obj(data[i], ',', properties);
|
||||
|
||||
let id = obj.id
|
||||
|
||||
if (id in ids.portals) {
|
||||
obj.portal = ids.portals[id];
|
||||
level_portals.push(obj);
|
||||
} else if (id in ids.orbs) {
|
||||
obj.orb = ids.orbs[id];
|
||||
|
||||
if (obj.orb in orb_array) {
|
||||
orb_array[obj.orb]++;
|
||||
} else {
|
||||
orb_array[obj.orb] = 1;
|
||||
}
|
||||
} else if (id in ids.triggers) {
|
||||
obj.trigger = ids.triggers[id];
|
||||
|
||||
if (obj.trigger in trigger_array) {
|
||||
trigger_array[obj.trigger]++;
|
||||
} else {
|
||||
trigger_array[obj.trigger] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.message) {
|
||||
level_text.push(obj)
|
||||
}
|
||||
|
||||
if (obj.triggerGroups) obj.triggerGroups.split('.').forEach(x => triggerGroups.push(x))
|
||||
if (obj.highDetail == 1) highDetail += 1
|
||||
|
||||
if (id in misc_objects) {
|
||||
const name = misc_objects[id];
|
||||
if (name in miscCounts) {
|
||||
miscCounts[name][0] += 1;
|
||||
} else {
|
||||
miscCounts[name] = [1, ids.misc[name][0]];
|
||||
}
|
||||
}
|
||||
|
||||
if (id in block_ids) {
|
||||
const name = block_ids[id];
|
||||
if (name in blockCounts) {
|
||||
blockCounts[name] += 1;
|
||||
} else {
|
||||
blockCounts[name] = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (obj.x) { // sometimes the field doesn't exist
|
||||
last = Math.max(last, obj.x);
|
||||
}
|
||||
|
||||
data[i] = obj;
|
||||
}
|
||||
|
||||
response.level = {
|
||||
name: level.name, id: level.id, author: level.author, authorID: level.authorID, accountID: level.accountID, large: level.large
|
||||
|
|
@ -82,23 +152,14 @@ zlib.unzip(levelString, (err, buffer) => {
|
|||
response.highDetail = highDetail
|
||||
response.settings = {}
|
||||
|
||||
response.portals = data.filter(x => x.portal).sort(function (a, b) {return parseInt(a.x) - parseInt(b.x)}).map(x => x.portal + " " + Math.floor(x.x / (Math.max(last, 529.0) + 340.0) * 100) + "%").join(", ")
|
||||
response.portals = level_portals.sort(function (a, b) {return parseInt(a.x) - parseInt(b.x)}).map(x => x.portal + " " + Math.floor(x.x / (Math.max(last, 529.0) + 340.0) * 100) + "%").join(", ")
|
||||
|
||||
response.orbs = {}
|
||||
orbArray = data.filter(x => x.orb).reduce( (a,b) => { //stolen from https://stackoverflow.com/questions/45064107/how-do-i-group-duplicate-objects-in-an-array
|
||||
var i = a.findIndex(x => x.orb === b.orb);
|
||||
return i === -1 ? a.push({ orb : b.orb, count : 1 }) : a[i].count++, a;
|
||||
}, []).sort(function (a, b) {return parseInt(b.count) - parseInt(a.count)})
|
||||
orbArray.forEach(x => response.orbs[x.orb] = x.count)
|
||||
response.orbs.total = data.filter(x => x.orb).length
|
||||
response.orbs = orb_array
|
||||
response.orbs.total = Object.values(orb_array).reduce((a, x) => a + x, 0); // we already have an array of objects, use it
|
||||
|
||||
response.triggers = trigger_array
|
||||
response.triggers.total = Object.values(trigger_array).reduce((a, x) => a + x, 0);
|
||||
|
||||
response.triggers = {}
|
||||
triggerArray = data.filter(x => x.trigger).reduce( (a,b) => {
|
||||
var i = a.findIndex(x => x.trigger === b.trigger);
|
||||
return i === -1 ? a.push({ trigger : b.trigger, count : 1 }) : a[i].count++, a;
|
||||
}, []).sort(function (a, b) {return parseInt(b.count) - parseInt(a.count)})
|
||||
triggerArray.forEach(x => response.triggers[x.trigger] = x.count)
|
||||
response.triggers.total = data.filter(x => x.trigger).length
|
||||
response.triggerGroups = {}
|
||||
response.blocks = sortObj(blockCounts)
|
||||
response.misc = sortObj(miscCounts, '0')
|
||||
|
|
@ -112,100 +173,123 @@ zlib.unzip(levelString, (err, buffer) => {
|
|||
response.triggerGroups = sortObj(response.triggerGroups)
|
||||
response.triggerGroups.total = Object.keys(response.triggerGroups).length
|
||||
|
||||
Object.keys(data[0]).forEach(x => {
|
||||
response.text = level_text.sort(function (a, b) {return parseInt(a.x) - parseInt(b.x)}).map(x => [Buffer.from(x.message, 'base64').toString(), Math.round(x.x / last * 99) + "%"])
|
||||
|
||||
const header_response = parse_header(header);
|
||||
response.settings = header_response.settings;
|
||||
response.colors = header_response.colors;
|
||||
|
||||
response.dataLength = rawData.length
|
||||
response.data = rawData
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
function parse_header(header) {
|
||||
let response = {};
|
||||
response.settings = {};
|
||||
response.colors = [];
|
||||
|
||||
const header_keyed = parse_obj(header, ',', init.values, true);
|
||||
|
||||
Object.keys(header_keyed).forEach(x => {
|
||||
let val = init.values[x]
|
||||
let name = val[0]
|
||||
let property = data[0][x]
|
||||
if (val[1] == "list") val = init[(val[0] + "s")][property]
|
||||
else if (val[1] == "number") val = Number(property)
|
||||
else if (val[1] == "bump") val = Number(property) + 1
|
||||
else if (val[1] == "bool") val = property != "0"
|
||||
let property = header_keyed[x]
|
||||
switch (val[1]) {
|
||||
case 'list':
|
||||
val = init[(val[0] + "s")][property];
|
||||
break;
|
||||
case 'number':
|
||||
val = Number(property);
|
||||
break;
|
||||
case 'bump':
|
||||
val = Number(property) + 1;
|
||||
break;
|
||||
case 'bool':
|
||||
val = property != "0";
|
||||
break;
|
||||
case 'extra-legacy-color': { // scope?
|
||||
// you can only imagine my fear when i discovered this was a thing
|
||||
// these literally are keys set the value, and to convert this to the color list we have to do this fun messy thing that shouldn't exist
|
||||
// since i wrote the 1.9 color before this, a lot of explaination will be there instead
|
||||
const colorInfo = name.split('-');
|
||||
const color = colorInfo[2]; // r,g,b
|
||||
const channel = colorInfo[1];
|
||||
|
||||
// you can only imagine my fear when i discovered this was a thing
|
||||
// these literally are keys set the value, and to convert this to the color list we have to do this fun messy thing that shouldn't exist
|
||||
else if (val[1] == 'extra-legacy-color') {
|
||||
// since i wrote the 1.9 color before this, a lot of explaination will be there instead
|
||||
const colorInfo = name.split('-');
|
||||
const color = colorInfo[2]; // r,g,b
|
||||
const channel = colorInfo[1];
|
||||
|
||||
if (color == 'r') {
|
||||
// first we create the color object
|
||||
response.colors.push({"channel": channel, "opacity": 1});
|
||||
}
|
||||
// from here we touch the color object
|
||||
let currentChannel = response.colors.find(k => k.channel == channel);
|
||||
if (color == 'blend') {
|
||||
currentChannel.blending = true; // only one color has blending though lol
|
||||
} else if (color == 'pcol' && property != 0) {
|
||||
currentChannel.pColor = property;
|
||||
}
|
||||
currentChannel[color] = property;
|
||||
}
|
||||
|
||||
// if a level has a legacy color, we can assume that it does not have a kS38 at all
|
||||
else if (val[1] == "legacy-color") {
|
||||
color = app.parseResponse(property, "_");
|
||||
|
||||
const keys = Object.keys(color)
|
||||
let colorObj = {}
|
||||
|
||||
// so here we parse the color to something understandable by the rest
|
||||
// slightly smart naming but it is also pretty gross
|
||||
// in a sense - the name would be something like legacy-G -> G
|
||||
const colorVal = name.split('-').pop()
|
||||
|
||||
keys.forEach(k => {if (colorStuff.properties[k]) colorObj[colorStuff.properties[k]] = color[k]})
|
||||
|
||||
colorObj.channel = colorVal
|
||||
|
||||
// from here stuff can continue as normal, ish
|
||||
if (colorObj.pColor == "-1" || colorObj.pColor == "0") delete colorObj.pColor;
|
||||
colorObj.opacity = 1; // 1.9 colors don't have this!
|
||||
if (colorObj.blending && colorObj.blending == '1') colorObj.blending = true; // 1.9 colors manage to always think they're blending - they're not
|
||||
else delete colorObj.blending;
|
||||
|
||||
if (colorVal == '3DL') { response.colors.splice(4, 0, colorObj); } // hardcode the position of 3DL, it typically goes at the end due to how RobTop make the headers
|
||||
else if (colorVal == 'Line') { colorObj.blending = true; response.colors.push(colorObj); } // in line with 2.1 behavior
|
||||
else { response.colors.push(colorObj); } // bruh whatever was done to make the color list originally was long
|
||||
}
|
||||
|
||||
else if (val[1] == "colors") {
|
||||
let colorList = property.split("|")
|
||||
colorList.forEach((x, y) => {
|
||||
color = app.parseResponse(x, "_")
|
||||
let keys = Object.keys(color)
|
||||
let colorObj = {}
|
||||
if (!color['6']) return colorList = colorList.filter((h, i) => y != i)
|
||||
|
||||
keys.forEach(k => {if (colorStuff.properties[k]) colorObj[colorStuff.properties[k]] = color[k]})
|
||||
if (colorStuff.channels[colorObj.channel]) colorObj.channel = colorStuff.channels[colorObj.channel]
|
||||
if (colorObj.channel > 1000) return;
|
||||
if (colorStuff.channels[colorObj.copiedChannel]) colorObj.copiedChannel = colorStuff.channels[colorObj.copiedChannel]
|
||||
if (colorObj.copiedChannel > 1000) delete colorObj.copiedChannel;
|
||||
if (colorObj.pColor == "-1") delete colorObj.pColor
|
||||
if (colorObj.blending) colorObj.blending = true
|
||||
if (colorObj.copiedHSV) {
|
||||
let hsv = colorObj.copiedHSV.split("a")
|
||||
colorObj.copiedHSV = {}
|
||||
hsv.forEach((x, y) => { colorObj.copiedHSV[colorStuff.hsv[y]] = x })
|
||||
colorObj.copiedHSV['s-checked'] = colorObj.copiedHSV['s-checked'] == 1
|
||||
colorObj.copiedHSV['v-checked'] = colorObj.copiedHSV['v-checked'] == 1
|
||||
if (colorObj.copyOpacity == 1) colorObj.copyOpacity = true
|
||||
if (color == 'r') {
|
||||
// first we create the color object
|
||||
response.colors.push({"channel": channel, "opacity": 1});
|
||||
}
|
||||
colorObj.opacity = +Number(colorObj.opacity).toFixed(2)
|
||||
colorList[y] = colorObj
|
||||
// from here we touch the color object
|
||||
let currentChannel = response.colors.find(k => k.channel == channel);
|
||||
if (color == 'blend') {
|
||||
currentChannel.blending = true; // only one color has blending though lol
|
||||
} else if (color == 'pcol' && property != 0) {
|
||||
currentChannel.pColor = property;
|
||||
}
|
||||
currentChannel[color] = property;
|
||||
break;
|
||||
}
|
||||
case 'legacy-color': {
|
||||
// if a level has a legacy color, we can assume that it does not have a kS38 at all
|
||||
const color = parse_obj(property, "_", colorStuff.properties);
|
||||
|
||||
let colorObj = color
|
||||
|
||||
// so here we parse the color to something understandable by the rest
|
||||
// slightly smart naming but it is also pretty gross
|
||||
// in a sense - the name would be something like legacy-G -> G
|
||||
const colorVal = name.split('-').pop()
|
||||
|
||||
colorObj.channel = colorVal
|
||||
|
||||
// from here stuff can continue as normal, ish
|
||||
if (colorObj.pColor == "-1" || colorObj.pColor == "0") delete colorObj.pColor;
|
||||
colorObj.opacity = 1; // 1.9 colors don't have this!
|
||||
if (colorObj.blending && colorObj.blending == '1') colorObj.blending = true; // 1.9 colors manage to always think they're blending - they're not
|
||||
else delete colorObj.blending;
|
||||
|
||||
if (colorVal == '3DL') { response.colors.splice(4, 0, colorObj); } // hardcode the position of 3DL, it typically goes at the end due to how RobTop make the headers
|
||||
else if (colorVal == 'Line') { colorObj.blending = true; response.colors.push(colorObj); } // in line with 2.1 behavior
|
||||
else { response.colors.push(colorObj); } // bruh whatever was done to make the color list originally was long
|
||||
break;
|
||||
}
|
||||
case 'colors': {
|
||||
let colorList = property.split("|")
|
||||
colorList.forEach((x, y) => {
|
||||
const color = parse_obj(x, "_", colorStuff.properties)
|
||||
let colorObj = color
|
||||
if (!color.channel) return colorList = colorList.filter((h, i) => y != i)
|
||||
|
||||
if (colorStuff.channels[colorObj.channel]) colorObj.channel = colorStuff.channels[colorObj.channel]
|
||||
if (colorObj.channel > 1000) return;
|
||||
if (colorStuff.channels[colorObj.copiedChannel]) colorObj.copiedChannel = colorStuff.channels[colorObj.copiedChannel]
|
||||
if (colorObj.copiedChannel > 1000) delete colorObj.copiedChannel;
|
||||
if (colorObj.pColor == "-1") delete colorObj.pColor
|
||||
if (colorObj.blending) colorObj.blending = true
|
||||
if (colorObj.copiedHSV) {
|
||||
let hsv = colorObj.copiedHSV.split("a")
|
||||
colorObj.copiedHSV = {}
|
||||
hsv.forEach((x, y) => { colorObj.copiedHSV[colorStuff.hsv[y]] = x })
|
||||
colorObj.copiedHSV['s-checked'] = colorObj.copiedHSV['s-checked'] == 1
|
||||
colorObj.copiedHSV['v-checked'] = colorObj.copiedHSV['v-checked'] == 1
|
||||
if (colorObj.copyOpacity == 1) colorObj.copyOpacity = true
|
||||
}
|
||||
colorObj.opacity = +Number(colorObj.opacity).toFixed(2)
|
||||
colorList[y] = colorObj
|
||||
});
|
||||
// we assume this is only going to be run once so... some stuff can go here
|
||||
colorList = colorList.filter(x => typeof x == "object")
|
||||
if (!colorList.find(x => x.channel == "Obj")) colorList.push({"r": "255", "g": "255", "b": "255", "channel": "Obj", "opacity": "1"})
|
||||
|
||||
let specialSort = ["BG", "G", "G2", "Line", "Obj", "3DL"]
|
||||
const specialSort = ["BG", "G", "G2", "Line", "Obj", "3DL"]
|
||||
let specialColors = colorList.filter(x => isNaN(x.channel)).sort(function (a, b) {return specialSort.indexOf( a.channel ) > specialSort.indexOf( b.channel ) } )
|
||||
let regularColors = colorList.filter(x => !isNaN(x.channel)).sort(function(a, b) {return (+a.channel) - (+b.channel) } );
|
||||
response.colors = specialColors.concat(regularColors)
|
||||
})
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
response.settings[name] = val
|
||||
})
|
||||
|
||||
|
|
@ -221,12 +305,6 @@ zlib.unzip(levelString, (err, buffer) => {
|
|||
if (k.includes('legacy')) delete response.settings[k];
|
||||
});
|
||||
|
||||
delete response.settings['colors']
|
||||
response.text = data.filter(x => x.message).sort(function (a, b) {return parseInt(a.x) - parseInt(b.x)}).map(x => [Buffer.from(x.message, 'base64').toString(), Math.round(x.x / last * 99) + "%"])
|
||||
response.dataLength = rawData.length
|
||||
response.data = rawData
|
||||
|
||||
return res.send(response)
|
||||
}
|
||||
|
||||
delete response.settings['colors'];
|
||||
return response;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue