diff options
Diffstat (limited to '')
| -rw-r--r-- | node_modules/iconv-lite/encodings/dbcs-codec.js | 532 |
1 files changed, 532 insertions, 0 deletions
diff --git a/node_modules/iconv-lite/encodings/dbcs-codec.js b/node_modules/iconv-lite/encodings/dbcs-codec.js new file mode 100644 index 0000000..bfec7f2 --- /dev/null +++ b/node_modules/iconv-lite/encodings/dbcs-codec.js @@ -0,0 +1,532 @@ +"use strict" +var Buffer = require("safer-buffer").Buffer + +// Multibyte codec. In this scheme, a character is represented by 1 or more bytes. +// Our codec supports UTF-16 surrogates, extensions for GB18030 and unicode sequences. +// To save memory and loading time, we read table files only when requested. + +exports._dbcs = DBCSCodec + +var UNASSIGNED = -1 +var GB18030_CODE = -2 +var SEQ_START = -10 +var NODE_START = -1000 +var UNASSIGNED_NODE = new Array(0x100) +var DEF_CHAR = -1 + +for (var i = 0; i < 0x100; i++) { UNASSIGNED_NODE[i] = UNASSIGNED } + +// Class DBCSCodec reads and initializes mapping tables. +function DBCSCodec (codecOptions, iconv) { + this.encodingName = codecOptions.encodingName + if (!codecOptions) { throw new Error("DBCS codec is called without the data.") } + if (!codecOptions.table) { throw new Error("Encoding '" + this.encodingName + "' has no data.") } + + // Load tables. + var mappingTable = codecOptions.table() + + // Decode tables: MBCS -> Unicode. + + // decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256. + // Trie root is decodeTables[0]. + // Values: >= 0 -> unicode character code. can be > 0xFFFF + // == UNASSIGNED -> unknown/unassigned sequence. + // == GB18030_CODE -> this is the end of a GB18030 4-byte sequence. + // <= NODE_START -> index of the next node in our trie to process next byte. + // <= SEQ_START -> index of the start of a character code sequence, in decodeTableSeq. + this.decodeTables = [] + this.decodeTables[0] = UNASSIGNED_NODE.slice(0) // Create root node. + + // Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here. + this.decodeTableSeq = [] + + // Actual mapping tables consist of chunks. Use them to fill up decode tables. + for (var i = 0; i < mappingTable.length; i++) { this._addDecodeChunk(mappingTable[i]) } + + // Load & create GB18030 tables when needed. + if (typeof codecOptions.gb18030 === "function") { + this.gb18030 = codecOptions.gb18030() // Load GB18030 ranges. + + // Add GB18030 common decode nodes. + var commonThirdByteNodeIdx = this.decodeTables.length + this.decodeTables.push(UNASSIGNED_NODE.slice(0)) + + var commonFourthByteNodeIdx = this.decodeTables.length + this.decodeTables.push(UNASSIGNED_NODE.slice(0)) + + // Fill out the tree + var firstByteNode = this.decodeTables[0] + for (var i = 0x81; i <= 0xFE; i++) { + var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]] + for (var j = 0x30; j <= 0x39; j++) { + if (secondByteNode[j] === UNASSIGNED) { + secondByteNode[j] = NODE_START - commonThirdByteNodeIdx + } else if (secondByteNode[j] > NODE_START) { + throw new Error("gb18030 decode tables conflict at byte 2") + } + + var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]] + for (var k = 0x81; k <= 0xFE; k++) { + if (thirdByteNode[k] === UNASSIGNED) { + thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx + } else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) { + continue + } else if (thirdByteNode[k] > NODE_START) { + throw new Error("gb18030 decode tables conflict at byte 3") + } + + var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]] + for (var l = 0x30; l <= 0x39; l++) { + if (fourthByteNode[l] === UNASSIGNED) { fourthByteNode[l] = GB18030_CODE } + } + } + } + } + } + + this.defaultCharUnicode = iconv.defaultCharUnicode + + // Encode tables: Unicode -> DBCS. + + // `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance. + // Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null. + // Values: >= 0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.). + // == UNASSIGNED -> no conversion found. Output a default char. + // <= SEQ_START -> it's an index in encodeTableSeq, see below. The character starts a sequence. + this.encodeTable = [] + + // `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of + // objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key + // means end of sequence (needed when one sequence is a strict subsequence of another). + // Objects are kept separately from encodeTable to increase performance. + this.encodeTableSeq = [] + + // Some chars can be decoded, but need not be encoded. + var skipEncodeChars = {} + if (codecOptions.encodeSkipVals) { + for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) { + var val = codecOptions.encodeSkipVals[i] + if (typeof val === "number") { skipEncodeChars[val] = true } else { + for (var j = val.from; j <= val.to; j++) { skipEncodeChars[j] = true } + } + } + } + + // Use decode trie to recursively fill out encode tables. + this._fillEncodeTable(0, 0, skipEncodeChars) + + // Add more encoding pairs when needed. + if (codecOptions.encodeAdd) { + for (var uChar in codecOptions.encodeAdd) { + if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar)) { this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]) } + } + } + + this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)] + if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]["?"] + if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0) +} + +DBCSCodec.prototype.encoder = DBCSEncoder +DBCSCodec.prototype.decoder = DBCSDecoder + +// Decoder helpers +DBCSCodec.prototype._getDecodeTrieNode = function (addr) { + var bytes = [] + for (; addr > 0; addr >>>= 8) { bytes.push(addr & 0xFF) } + if (bytes.length == 0) { bytes.push(0) } + + var node = this.decodeTables[0] + for (var i = bytes.length - 1; i > 0; i--) { // Traverse nodes deeper into the trie. + var val = node[bytes[i]] + + if (val == UNASSIGNED) { // Create new node. + node[bytes[i]] = NODE_START - this.decodeTables.length + this.decodeTables.push(node = UNASSIGNED_NODE.slice(0)) + } else if (val <= NODE_START) { // Existing node. + node = this.decodeTables[NODE_START - val] + } else { throw new Error("Overwrite byte in " + this.encodingName + ", addr: " + addr.toString(16)) } + } + return node +} + +DBCSCodec.prototype._addDecodeChunk = function (chunk) { + // First element of chunk is the hex mbcs code where we start. + var curAddr = parseInt(chunk[0], 16) + + // Choose the decoding node where we'll write our chars. + var writeTable = this._getDecodeTrieNode(curAddr) + curAddr = curAddr & 0xFF + + // Write all other elements of the chunk to the table. + for (var k = 1; k < chunk.length; k++) { + var part = chunk[k] + if (typeof part === "string") { // String, write as-is. + for (var l = 0; l < part.length;) { + var code = part.charCodeAt(l++) + if (code >= 0xD800 && code < 0xDC00) { // Decode surrogate + var codeTrail = part.charCodeAt(l++) + if (codeTrail >= 0xDC00 && codeTrail < 0xE000) { writeTable[curAddr++] = 0x10000 + (code - 0xD800) * 0x400 + (codeTrail - 0xDC00) } else { throw new Error("Incorrect surrogate pair in " + this.encodingName + " at chunk " + chunk[0]) } + } else if (code > 0x0FF0 && code <= 0x0FFF) { // Character sequence (our own encoding used) + var len = 0xFFF - code + 2 + var seq = [] + for (var m = 0; m < len; m++) { seq.push(part.charCodeAt(l++)) } // Simple variation: don't support surrogates or subsequences in seq. + + writeTable[curAddr++] = SEQ_START - this.decodeTableSeq.length + this.decodeTableSeq.push(seq) + } else { writeTable[curAddr++] = code } // Basic char + } + } else if (typeof part === "number") { // Integer, meaning increasing sequence starting with prev character. + var charCode = writeTable[curAddr - 1] + 1 + for (var l = 0; l < part; l++) { writeTable[curAddr++] = charCode++ } + } else { throw new Error("Incorrect type '" + typeof part + "' given in " + this.encodingName + " at chunk " + chunk[0]) } + } + if (curAddr > 0xFF) { throw new Error("Incorrect chunk in " + this.encodingName + " at addr " + chunk[0] + ": too long" + curAddr) } +} + +// Encoder helpers +DBCSCodec.prototype._getEncodeBucket = function (uCode) { + var high = uCode >> 8 // This could be > 0xFF because of astral characters. + if (this.encodeTable[high] === undefined) { + this.encodeTable[high] = UNASSIGNED_NODE.slice(0) + } // Create bucket on demand. + return this.encodeTable[high] +} + +DBCSCodec.prototype._setEncodeChar = function (uCode, dbcsCode) { + var bucket = this._getEncodeBucket(uCode) + var low = uCode & 0xFF + if (bucket[low] <= SEQ_START) { this.encodeTableSeq[SEQ_START - bucket[low]][DEF_CHAR] = dbcsCode } // There's already a sequence, set a single-char subsequence of it. + else if (bucket[low] == UNASSIGNED) { bucket[low] = dbcsCode } +} + +DBCSCodec.prototype._setEncodeSequence = function (seq, dbcsCode) { + // Get the root of character tree according to first character of the sequence. + var uCode = seq[0] + var bucket = this._getEncodeBucket(uCode) + var low = uCode & 0xFF + + var node + if (bucket[low] <= SEQ_START) { + // There's already a sequence with - use it. + node = this.encodeTableSeq[SEQ_START - bucket[low]] + } else { + // There was no sequence object - allocate a new one. + node = {} + if (bucket[low] !== UNASSIGNED) node[DEF_CHAR] = bucket[low] // If a char was set before - make it a single-char subsequence. + bucket[low] = SEQ_START - this.encodeTableSeq.length + this.encodeTableSeq.push(node) + } + + // Traverse the character tree, allocating new nodes as needed. + for (var j = 1; j < seq.length - 1; j++) { + var oldVal = node[uCode] + if (typeof oldVal === "object") { node = oldVal } else { + node = node[uCode] = {} + if (oldVal !== undefined) { node[DEF_CHAR] = oldVal } + } + } + + // Set the leaf to given dbcsCode. + uCode = seq[seq.length - 1] + node[uCode] = dbcsCode +} + +DBCSCodec.prototype._fillEncodeTable = function (nodeIdx, prefix, skipEncodeChars) { + var node = this.decodeTables[nodeIdx] + var hasValues = false + var subNodeEmpty = {} + for (var i = 0; i < 0x100; i++) { + var uCode = node[i] + var mbCode = prefix + i + if (skipEncodeChars[mbCode]) { continue } + + if (uCode >= 0) { + this._setEncodeChar(uCode, mbCode) + hasValues = true + } else if (uCode <= NODE_START) { + var subNodeIdx = NODE_START - uCode + if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030). + var newPrefix = (mbCode << 8) >>> 0 // NOTE: '>>> 0' keeps 32-bit num positive. + if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars)) { hasValues = true } else { subNodeEmpty[subNodeIdx] = true } + } + } else if (uCode <= SEQ_START) { + this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode) + hasValues = true + } + } + return hasValues +} + +// == Encoder ================================================================== + +function DBCSEncoder (options, codec) { + // Encoder state + this.leadSurrogate = -1 + this.seqObj = undefined + + // Static data + this.encodeTable = codec.encodeTable + this.encodeTableSeq = codec.encodeTableSeq + this.defaultCharSingleByte = codec.defCharSB + this.gb18030 = codec.gb18030 +} + +DBCSEncoder.prototype.write = function (str) { + var newBuf = Buffer.alloc(str.length * (this.gb18030 ? 4 : 3)) + var leadSurrogate = this.leadSurrogate + var seqObj = this.seqObj + var nextChar = -1 + var i = 0; var j = 0 + + while (true) { + // 0. Get next character. + if (nextChar === -1) { + if (i == str.length) break + var uCode = str.charCodeAt(i++) + } else { + var uCode = nextChar + nextChar = -1 + } + + // 1. Handle surrogates. + if (uCode >= 0xD800 && uCode < 0xE000) { // Char is one of surrogates. + if (uCode < 0xDC00) { // We've got lead surrogate. + if (leadSurrogate === -1) { + leadSurrogate = uCode + continue + } else { + leadSurrogate = uCode + // Double lead surrogate found. + uCode = UNASSIGNED + } + } else { // We've got trail surrogate. + if (leadSurrogate !== -1) { + uCode = 0x10000 + (leadSurrogate - 0xD800) * 0x400 + (uCode - 0xDC00) + leadSurrogate = -1 + } else { + // Incomplete surrogate pair - only trail surrogate found. + uCode = UNASSIGNED + } + } + } else if (leadSurrogate !== -1) { + // Incomplete surrogate pair - only lead surrogate found. + nextChar = uCode; uCode = UNASSIGNED // Write an error, then current char. + leadSurrogate = -1 + } + + // 2. Convert uCode character. + var dbcsCode = UNASSIGNED + if (seqObj !== undefined && uCode != UNASSIGNED) { // We are in the middle of the sequence + var resCode = seqObj[uCode] + if (typeof resCode === "object") { // Sequence continues. + seqObj = resCode + continue + } else if (typeof resCode === "number") { // Sequence finished. Write it. + dbcsCode = resCode + } else if (resCode == undefined) { // Current character is not part of the sequence. + // Try default character for this sequence + resCode = seqObj[DEF_CHAR] + if (resCode !== undefined) { + dbcsCode = resCode // Found. Write it. + nextChar = uCode // Current character will be written too in the next iteration. + } else { + // TODO: What if we have no default? (resCode == undefined) + // Then, we should write first char of the sequence as-is and try the rest recursively. + // Didn't do it for now because no encoding has this situation yet. + // Currently, just skip the sequence and write current char. + } + } + seqObj = undefined + } else if (uCode >= 0) { // Regular character + var subtable = this.encodeTable[uCode >> 8] + if (subtable !== undefined) { dbcsCode = subtable[uCode & 0xFF] } + + if (dbcsCode <= SEQ_START) { // Sequence start + seqObj = this.encodeTableSeq[SEQ_START - dbcsCode] + continue + } + + if (dbcsCode == UNASSIGNED && this.gb18030) { + // Use GB18030 algorithm to find character(s) to write. + var idx = findIdx(this.gb18030.uChars, uCode) + if (idx != -1) { + var dbcsCode = this.gb18030.gbChars[idx] + (uCode - this.gb18030.uChars[idx]) + newBuf[j++] = 0x81 + Math.floor(dbcsCode / 12600); dbcsCode = dbcsCode % 12600 + newBuf[j++] = 0x30 + Math.floor(dbcsCode / 1260); dbcsCode = dbcsCode % 1260 + newBuf[j++] = 0x81 + Math.floor(dbcsCode / 10); dbcsCode = dbcsCode % 10 + newBuf[j++] = 0x30 + dbcsCode + continue + } + } + } + + // 3. Write dbcsCode character. + if (dbcsCode === UNASSIGNED) { dbcsCode = this.defaultCharSingleByte } + + if (dbcsCode < 0x100) { + newBuf[j++] = dbcsCode + } else if (dbcsCode < 0x10000) { + newBuf[j++] = dbcsCode >> 8 // high byte + newBuf[j++] = dbcsCode & 0xFF // low byte + } else if (dbcsCode < 0x1000000) { + newBuf[j++] = dbcsCode >> 16 + newBuf[j++] = (dbcsCode >> 8) & 0xFF + newBuf[j++] = dbcsCode & 0xFF + } else { + newBuf[j++] = dbcsCode >>> 24 + newBuf[j++] = (dbcsCode >>> 16) & 0xFF + newBuf[j++] = (dbcsCode >>> 8) & 0xFF + newBuf[j++] = dbcsCode & 0xFF + } + } + + this.seqObj = seqObj + this.leadSurrogate = leadSurrogate + return newBuf.slice(0, j) +} + +DBCSEncoder.prototype.end = function () { + if (this.leadSurrogate === -1 && this.seqObj === undefined) { return } // All clean. Most often case. + + var newBuf = Buffer.alloc(10); var j = 0 + + if (this.seqObj) { // We're in the sequence. + var dbcsCode = this.seqObj[DEF_CHAR] + if (dbcsCode !== undefined) { // Write beginning of the sequence. + if (dbcsCode < 0x100) { + newBuf[j++] = dbcsCode + } else { + newBuf[j++] = dbcsCode >> 8 // high byte + newBuf[j++] = dbcsCode & 0xFF // low byte + } + } else { + // See todo above. + } + this.seqObj = undefined + } + + if (this.leadSurrogate !== -1) { + // Incomplete surrogate pair - only lead surrogate found. + newBuf[j++] = this.defaultCharSingleByte + this.leadSurrogate = -1 + } + + return newBuf.slice(0, j) +} + +// Export for testing +DBCSEncoder.prototype.findIdx = findIdx + +// == Decoder ================================================================== + +function DBCSDecoder (options, codec) { + // Decoder state + this.nodeIdx = 0 + this.prevBytes = [] + + // Static data + this.decodeTables = codec.decodeTables + this.decodeTableSeq = codec.decodeTableSeq + this.defaultCharUnicode = codec.defaultCharUnicode + this.gb18030 = codec.gb18030 +} + +DBCSDecoder.prototype.write = function (buf) { + var newBuf = Buffer.alloc(buf.length * 2) + var nodeIdx = this.nodeIdx + var prevBytes = this.prevBytes; var prevOffset = this.prevBytes.length + var seqStart = -this.prevBytes.length // idx of the start of current parsed sequence. + var uCode + + for (var i = 0, j = 0; i < buf.length; i++) { + var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset] + + // Lookup in current trie node. + var uCode = this.decodeTables[nodeIdx][curByte] + + if (uCode >= 0) { + // Normal character, just use it. + } else if (uCode === UNASSIGNED) { // Unknown char. + // TODO: Callback with seq. + uCode = this.defaultCharUnicode.charCodeAt(0) + i = seqStart // Skip one byte ('i' will be incremented by the for loop) and try to parse again. + } else if (uCode === GB18030_CODE) { + if (i >= 3) { + var ptr = (buf[i - 3] - 0x81) * 12600 + (buf[i - 2] - 0x30) * 1260 + (buf[i - 1] - 0x81) * 10 + (curByte - 0x30) + } else { + var ptr = (prevBytes[i - 3 + prevOffset] - 0x81) * 12600 + + (((i - 2 >= 0) ? buf[i - 2] : prevBytes[i - 2 + prevOffset]) - 0x30) * 1260 + + (((i - 1 >= 0) ? buf[i - 1] : prevBytes[i - 1 + prevOffset]) - 0x81) * 10 + + (curByte - 0x30) + } + var idx = findIdx(this.gb18030.gbChars, ptr) + uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx] + } else if (uCode <= NODE_START) { // Go to next trie node. + nodeIdx = NODE_START - uCode + continue + } else if (uCode <= SEQ_START) { // Output a sequence of chars. + var seq = this.decodeTableSeq[SEQ_START - uCode] + for (var k = 0; k < seq.length - 1; k++) { + uCode = seq[k] + newBuf[j++] = uCode & 0xFF + newBuf[j++] = uCode >> 8 + } + uCode = seq[seq.length - 1] + } else { throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte) } + + // Write the character to buffer, handling higher planes using surrogate pair. + if (uCode >= 0x10000) { + uCode -= 0x10000 + var uCodeLead = 0xD800 | (uCode >> 10) + newBuf[j++] = uCodeLead & 0xFF + newBuf[j++] = uCodeLead >> 8 + + uCode = 0xDC00 | (uCode & 0x3FF) + } + newBuf[j++] = uCode & 0xFF + newBuf[j++] = uCode >> 8 + + // Reset trie node. + nodeIdx = 0; seqStart = i + 1 + } + + this.nodeIdx = nodeIdx + this.prevBytes = (seqStart >= 0) + ? Array.prototype.slice.call(buf, seqStart) + : prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf)) + + return newBuf.slice(0, j).toString("ucs2") +} + +DBCSDecoder.prototype.end = function () { + var ret = "" + + // Try to parse all remaining chars. + while (this.prevBytes.length > 0) { + // Skip 1 character in the buffer. + ret += this.defaultCharUnicode + var bytesArr = this.prevBytes.slice(1) + + // Parse remaining as usual. + this.prevBytes = [] + this.nodeIdx = 0 + if (bytesArr.length > 0) { ret += this.write(bytesArr) } + } + + this.prevBytes = [] + this.nodeIdx = 0 + return ret +} + +// Binary search for GB18030. Returns largest i such that table[i] <= val. +function findIdx (table, val) { + if (table[0] > val) { return -1 } + + var l = 0; var r = table.length + while (l < r - 1) { // always table[l] <= val < table[r] + var mid = l + ((r - l + 1) >> 1) + if (table[mid] <= val) { l = mid } else { r = mid } + } + return l +} |