Transaction: G7KJgZr...HmHB

{ "function_name": "bitmap_v2", "code": "\n const startTime = Date.now();\n\n function indexOfFirstBitInByteArray(bytes, startBit) {\n let firstBit = startBit % 8;\n for (let iByte = Math.floor(startBit / 8); iByte < bytes.length; iByte++) {\n if (bytes[iByte] > 0) {\n for (let iBit = firstBit; iBit <= 7; iBit++) {\n if (bytes[iByte] & (1 << (7 - iBit))) {\n return iByte * 8 + iBit;\n }\n }\n }\n firstBit = 0;\n }\n return -1;\n }\n\n function setBitInBitmap(uint8Array, bit, bitValue = true) {\n if (!bitValue) return uint8Array;\n const newLen = Math.floor(bit / 8) + 1;\n let result = uint8Array;\n if (uint8Array.length < newLen) {\n result = new Uint8Array(new ArrayBuffer(newLen));\n result.set(uint8Array);\n }\n result[Math.floor(bit / 8)] |= 1 << (7 - (bit % 8));\n return result;\n }\n\n function getBitInByteArray(bytes, bitIndex) {\n const b = Math.floor(bitIndex / 8);\n const bi = bitIndex % 8;\n return (bytes[b] & (1 << (7 - bi))) > 0;\n }\n\n // takes numbers between [start, end] bits inclusive in byte array and\n // returns decimal number they represent\n function getNumberBetweenBits(bytes, start, end) {\n const len = end - start + 1;\n let r = 0;\n for (let i = start, rbit = 0; i <= end; i++, rbit++) {\n if (getBitInByteArray(bytes, i)) {\n r |= 1 << (len - 1 - rbit);\n }\n }\n return r;\n }\n\n // Writes Elias gamma coding bits for number x into result bytes array starting with index startBit.\n // Returns index of the next bit after the coding.\n // Examples: https://en.wikipedia.org/wiki/Elias_gamma_coding\n function writeEliasGammaBits(x, result, startBit) {\n if (x === 0) return { bit: startBit, result };\n if (x === 1) {\n setBitInBitmap(result, startBit);\n return { bit: startBit + 1, result };\n }\n let bit = startBit;\n const N = Math.floor(Math.log2(x));\n const remainder = x - 2 ** N;\n bit += N;\n result = setBitInBitmap(result, bit++);\n for (let ri = 0; ri < N; ri++, bit++) {\n if (remainder & (1 << (N - 1 - ri))) {\n result = setBitInBitmap(result, bit);\n }\n }\n return { bit, result };\n }\n\n // stores first char (0 or 1) and then repeats alternating repeating sequences using Elias gamma coding\n // pads the resulting string at the end with '0's for length to be divisible by 8\n function compressBitmapArray(uint8Array) {\n let curBit = (uint8Array[0] & 0b10000000) > 0;\n let curBitStretch = 0;\n let resultBuffer = new ArrayBuffer(9000);\n let result = new Uint8Array(resultBuffer);\n let nextBit = 0;\n result = setBitInBitmap(result, nextBit++, curBit);\n for (let ibit = 0; ibit < uint8Array.length * 8; ibit++) {\n if (getBitInByteArray(uint8Array, ibit) === curBit) {\n curBitStretch++;\n } else {\n const w = writeEliasGammaBits(curBitStretch, result, nextBit);\n nextBit = w.bit;\n result = w.result;\n curBit = !curBit;\n curBitStretch = 1;\n }\n }\n const w = writeEliasGammaBits(curBitStretch, result, nextBit);\n nextBit = w.bit;\n result = w.result.slice(0, Math.ceil(nextBit / 8));\n return result;\n }\n\n // Returns first number x and corresponding coded bits length of the first occurrence of Elias gamma coding\n function decodeEliasGammaFirstEntryFromBytes(bytes, startBit = 0) {\n if (!bytes || bytes.length === 0) return { x: 0, lastBit: 0 };\n const idx = indexOfFirstBitInByteArray(bytes, startBit);\n if (idx < 0) {\n return { x: 0, len: bytes.length * 8 };\n }\n const N = idx - startBit;\n const remainder = getNumberBetweenBits(bytes, idx + 1, idx + N);\n return { x: 2 ** N + remainder, lastBit: idx + N };\n }\n\n // Decompresses Elias-gamma coded bytes to Uint8Array\n function decompressToBitmapArray(compressedBytes) {\n let curBit = (compressedBytes[0] & 0x80) > 0;\n const buffer = new ArrayBuffer(90000);\n let bufferLength = 0;\n let result = new Uint8Array(buffer);\n let compressedBitIdx = 1;\n let nextBitIdx = 0;\n while (compressedBitIdx < compressedBytes.length * 8) {\n const { x, lastBit } = decodeEliasGammaFirstEntryFromBytes(\n compressedBytes,\n compressedBitIdx\n );\n compressedBitIdx = lastBit + 1;\n if (bufferLength * 8 < nextBitIdx + x) {\n bufferLength = Math.ceil((nextBitIdx + x) / 8);\n }\n for (let i = 0; curBit && i < x; i++) {\n setBitInBitmap(result, nextBitIdx + i);\n }\n nextBitIdx += x;\n curBit = !curBit;\n if (x === 0) break; // we won't find any Elias gamma here, exiting\n }\n return result.slice(0, bufferLength);\n }\n\n function addIndexCompressed(compressedBase64, index) {\n const d = Date.now();\n const bitmap = decompressToBitmapArray(\n Buffer.from(compressedBase64, \"base64\")\n );\n const decompressMs = Date.now() - d;\n const s = Date.now();\n const newBitmap = setBitInBitmap(bitmap, index);\n const setsMs = Date.now() - s;\n const c = Date.now();\n const compressed = compressBitmapArray(newBitmap);\n const compMs = Date.now() - c;\n console.log(\n `decompressMs=${decompressMs}, setsMs=${setsMs}, compMs=${compMs}`\n );\n return Buffer.from(compressed).toString(\"base64\");\n }\n\n const blockDate = new Date(\n block.streamerMessage.block.header.timestamp / 1000000\n )\n .toISOString()\n .substring(0, 10);\n const actionsByReceiver = block.actions().reduce((groups, action) => {\n (groups[action.receiverId] ||= []).push(action);\n return groups;\n }, {});\n const allReceivers = Object.keys(actionsByReceiver);\n // console.log(`There are ${allReceivers.length} receivers in this block.`);\n // console.log(\n // `SELECT * FROM \"actions_index\" WHERE block_date='${blockDate}' AND receiver_id IN (${allReceivers\n // .map((r) => `'${r}'`)\n // .join(\",\")})`\n // );\n const currIndexes =\n (await context.db.ActionsIndex.select({\n block_date: blockDate,\n receiver_id: allReceivers,\n })) ?? [];\n const startTimeR = Date.now();\n const upserts = allReceivers.map((receiverId) => {\n const currentIndex = currIndexes.find((i) => i?.receiver_id === receiverId);\n const blockIndexInCurrentBitmap = currentIndex?.first_block_height\n ? block.blockHeight - currentIndex?.first_block_height\n : 0;\n\n const s = Date.now();\n const newBitmap = addIndexCompressed(\n currentIndex?.bitmap ?? \"\",\n blockIndexInCurrentBitmap\n );\n // console.log(`${receiverId}: ${Date.now() - s} ms, first_block_height: ${currentIndex?.first_block_height ?? 0}, blockIndexInCurrentBitmap: ${blockIndexInCurrentBitmap}, bitmapLen: ${(currentIndex?.bitmap ?? \"\").length}`)\n return {\n first_block_height: currentIndex?.first_block_height ?? block.blockHeight,\n block_date: blockDate,\n receiver_id: receiverId,\n bitmap: newBitmap,\n };\n });\n const endTimeR = Date.now();\n //console.log(\"upserts\", JSON.stringify(upserts));\n await context.db.ActionsIndex.upsert(\n upserts,\n [\"block_date\", \"receiver_id\"],\n [\"bitmap\"]\n );\n const endTime = Date.now();\n context.log(\n `Computing bitmaps for ${allReceivers.length} receivers took ${\n endTimeR - startTimeR\n }ms; total time ${endTime - startTime}ms`\n );\n", "schema": "CREATE TABLE\n \"actions_index\" (\n \"block_date\" TEXT NOT NULL,\n \"receiver_id\" TEXT NOT NULL,\n \"first_block_height\" NUMERIC(20) NOT NULL,\n \"bitmap\" TEXT NOT NULL,\n PRIMARY KEY (\"block_date\", \"receiver_id\")\n )\n", "start_block": "CONTINUE", "rule": { "kind": "ACTION_ANY", "affected_account_id": "*", "status": "SUCCESS" } }